Query engine with natural and virtual points (#13248)
* new query engine * use Index * Revert change that changed in-memory page indexing to start time - update_every + 1 * use internal_error() to cleanup the code * interpolates values when generating points Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Costa Tsaousis committed
Jun 29, 2022 at 19:24 UTC
2fc0aaca9a27520c874756b16d95794457ed6ac9
6 files changed
+387
-335
daemon/unit_test.c
+9
-4
@@ -1824,7 +1824,10 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
1824
long points = (time_end - time_start) / update_every;
1825
for (i = 0 ; i < CHARTS ; ++i) {
1826
ONEWAYALLOC *owa = onewayalloc_create(0);
1827
- RRDR *r = rrd2rrdr(owa, st[i], points, time_start, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, NULL, 0);
1827
+ RRDR *r = rrd2rrdr(owa, st[i], points, time_start, time_end,
1828
+ RRDR_GROUPING_AVERAGE, 0, RRDR_OPTION_NATURAL_POINTS,
1829
+ NULL, NULL, NULL, 0);
1830
+
1831
if (!r) {
1832
fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", st[i]->name, current_region);
1833
return ++errors;
@@ -1846,14 +1849,14 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
1849
1850
same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
1851
if(!same) {
1849
- if(value_errors < 10)
1852
+ if(value_errors < 20)
1853
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
1854
", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
1855
st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
1856
value_errors++;
1857
}
1858
if(time_retrieved != time_now) {
1856
- if(!time_errors)
1859
+ if(time_errors < 20)
1860
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",
1861
st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved);
1862
time_errors++;
@@ -1955,7 +1958,9 @@ int test_dbengine(void)
1958
long point_offset = (time_start[current_region] - time_start[0]) / update_every;
1959
for (i = 0 ; i < CHARTS ; ++i) {
1960
ONEWAYALLOC *owa = onewayalloc_create(0);
1958
- RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, NULL, 0);
1961
+ RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every,
1962
+ time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0,
1963
+ RRDR_OPTION_NATURAL_POINTS, NULL, NULL, NULL, 0);
1964
if (!r) {
1965
fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
1966
++errors;
database/metric_correlations.c
+3
-1
@@ -384,6 +384,8 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
384
long long points, RRDR_OPTIONS options,
385
RRDR_GROUPING group, const char *group_options,
386
uint32_t shifts, int timeout, MC_STATS *stats) {
387
+ options |= RRDR_OPTION_NATURAL_POINTS;
388
+
389
long group_time = 0;
390
struct context_param *context_param_list = NULL;
391
@@ -518,7 +520,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
520
long long after, long long before,
521
RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,
522
int timeout, MC_STATS *stats) {
521
- options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE;
523
+ options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS;
524
long group_time = 0;
525
526
int correlated_dimensions = 0;
database/ram/rrddim_mem.c
+1
-1
@@ -34,7 +34,7 @@ void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t st
34
h->slot_timestamp = rrdset_slot2time(rd->rrdset, h->slot);
35
h->last_timestamp = rrdset_slot2time(rd->rrdset, h->last_slot);
36
37
- // info("QUERY: start %ld, end %ld, next %ld, first %ld, last %ld", start_time, end_time, h->next_timestamp, h->slot_timestamp, h->last_timestamp);
37
+ // info("RRDDIM QUERY INIT: start %ld, end %ld, next %ld, first %ld, last %ld, dt %ld", start_time, end_time, h->next_timestamp, h->slot_timestamp, h->last_timestamp, h->dt);
38
39
handle->handle = (STORAGE_QUERY_HANDLE *)h;
40
}
web/api/queries/query.c
+350
-306
@@ -316,14 +316,12 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) {
316
}
317
if(!found) {
318
errno = 0;
319
-#ifdef NETDATA_INTERNAL_CHECKS
320
- error("INTERNAL ERROR: grouping method %u not found. Using 'average'", (unsigned int)group_method);
321
-#endif
322
- r->internal.grouping_create= grouping_create_average;
323
- r->internal.grouping_reset = grouping_reset_average;
324
- r->internal.grouping_free = grouping_free_average;
325
- r->internal.grouping_add = grouping_add_average;
326
- r->internal.grouping_flush = grouping_flush_average;
319
+ internal_error(true, "QUERY: grouping method %u not found. Using 'average'", (unsigned int)group_method);
320
+ r->internal.grouping_create = grouping_create_average;
321
+ r->internal.grouping_reset = grouping_reset_average;
322
+ r->internal.grouping_free = grouping_free_average;
323
+ r->internal.grouping_add = grouping_add_average;
324
+ r->internal.grouping_flush = grouping_flush_average;
325
}
326
}
327
@@ -406,15 +404,13 @@ static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rr
404
static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) {
405
rrdr_line++;
406
409
- #ifdef NETDATA_INTERNAL_CHECKS
410
-
411
- if(unlikely(rrdr_line >= r->n))
412
- error("INTERNAL ERROR: requested to step above RRDR size for chart '%s'", r->st->name);
407
+ internal_error(rrdr_line >= r->n,
408
+ "QUERY: requested to step above RRDR size for chart '%s'",
409
+ r->st->name);
410
414
- if(unlikely(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t))
415
- error("INTERNAL ERROR: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'", (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name);
416
-
417
- #endif
411
+ internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t,
412
+ "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'",
413
+ (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name);
414
415
// save the time
416
r->t[rrdr_line] = t;
@@ -430,6 +426,17 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) {
426
// ----------------------------------------------------------------------------
427
// fill RRDR for a single dimension
428
429
+static inline NETDATA_DOUBLE interpolate_value(NETDATA_DOUBLE this_value, NETDATA_DOUBLE last_value, time_t last_value_end_t, time_t this_value_start_t, time_t now, time_t this_value_end_t) {
430
+ if(unlikely(
431
+ this_value_start_t + 1 == this_value_end_t ||
432
+ !netdata_double_isnumber(this_value) ||
433
+ !netdata_double_isnumber(last_value) ||
434
+ last_value_end_t != this_value_start_t))
435
+ return this_value;
436
+
437
+ return last_value + (this_value - last_value) * ( 1.0 - (NETDATA_DOUBLE)(this_value_end_t - now) / (NETDATA_DOUBLE)(this_value_end_t - this_value_start_t) );
438
+}
439
+
440
static inline void rrd2rrdr_do_dimension(
441
RRDR *r
442
, long points_wanted
@@ -437,12 +444,14 @@ static inline void rrd2rrdr_do_dimension(
444
, long dim_id_in_rrdr
445
, time_t after_wanted
446
, time_t before_wanted
440
- , uint32_t options
447
+ , RRDR_OPTIONS options
448
){
442
- time_t now = after_wanted,
443
- dt = r->update_every / r->group, /* usually is st->update_every */
444
- max_date = 0,
445
- min_date = 0;
449
+ time_t now = after_wanted,
450
+ query_granularity = r->update_every / r->group,
451
+ max_date = 0,
452
+ min_date = 0;
453
+
454
+ bool interpolate = query_granularity < rd->update_every;
455
456
long group_points_wanted = r->group,
457
points_added = 0, group_points_added = 0, group_points_non_zero = 0,
@@ -462,39 +471,57 @@ static inline void rrd2rrdr_do_dimension(
471
void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value) = r->internal.grouping_add;
472
NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) = r->internal.grouping_flush;
473
465
- NETDATA_DOUBLE last_point_value;
466
- SN_FLAGS last_point_flags;
467
- time_t last_point_start_time;
468
- time_t last_point_end_time;
469
- size_t last_point_anomaly;
474
+ NETDATA_DOUBLE last2_point_value;
475
+ //SN_FLAGS last2_point_flags;
476
+ //size_t last2_point_anomaly;
477
+ //time_t last2_point_start_time;
478
+ time_t last2_point_end_time;
479
+
480
+ NETDATA_DOUBLE last1_point_value = NAN;
481
+ SN_FLAGS last1_point_flags = SN_EMPTY_SLOT;
482
+ size_t last1_point_anomaly = 0;
483
+ time_t last1_point_start_time = 0;
484
+ time_t last1_point_end_time = 0;
485
486
NETDATA_DOUBLE new_point_value = NAN;
487
SN_FLAGS new_point_flags = SN_EMPTY_SLOT;
488
+ size_t new_point_anomaly = 0;
489
time_t new_point_start_time = 0;
490
time_t new_point_end_time = 0;
475
- size_t new_point_anomaly = 0;
491
477
- for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) {
492
+ for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += query_granularity) {
493
479
- // TODO - should be removed when before and after are always respected
480
- // independently of the databaase first and last time and points_wanted
481
- // is set to a sane number for the user to get the timeframe wanted.
482
- // Without the above, this check is needed to stop the loop when the
483
- // points_wanted is set to an unreasonably high number for the duration
484
- // of the query.
494
if(unlikely(now > before_wanted))
495
break;
496
488
- // save the old point, in case we need it
489
- last_point_value = new_point_value;
490
- last_point_flags = new_point_flags;
491
- last_point_anomaly = new_point_anomaly;
492
- last_point_start_time = new_point_start_time;
493
- last_point_end_time = new_point_end_time;
497
+ last2_point_value = last1_point_value;
498
+ //last2_point_flags = last1_point_flags;
499
+ //last2_point_anomaly = last1_point_anomaly;
500
+ //last2_point_start_time = last1_point_start_time;
501
+ last2_point_end_time = last1_point_end_time;
502
+
503
+ last1_point_value = new_point_value;
504
+ last1_point_flags = new_point_flags;
505
+ last1_point_anomaly = new_point_anomaly;
506
+ last1_point_start_time = new_point_start_time;
507
+ last1_point_end_time = new_point_end_time;
508
509
if(likely(!rd->state->query_ops.is_finished(&handle))) {
510
// fetch the new point
511
new_point_value = next_metric(&handle, &new_point_start_time, &new_point_end_time, &new_point_flags);
512
+ db_points_read++;
513
+
514
+ // dbengine does not take into account the starting time of points
515
+ // and depending on the data collection frequency it may return
516
+ // a point that is just before the wanted one.
517
+ // So, here we fetch the next one.
518
+ if(unlikely(new_point_end_time < now)) {
519
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before now (now = %ld, after_wanted = %ld, before_wanted = %ld, dt = %ld). Fetching the next one.",
520
+ rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time, now, after_wanted, before_wanted, query_granularity);
521
+
522
+ new_point_value = next_metric(&handle, &new_point_start_time, &new_point_end_time, &new_point_flags);
523
+ db_points_read++;
524
+ }
525
526
if(likely(netdata_double_isnumber(new_point_value))) {
527
new_point_anomaly = (new_point_flags & SN_ANOMALY_BIT) ? 0 : 100;
@@ -509,48 +536,57 @@ static inline void rrd2rrdr_do_dimension(
536
}
537
538
if(unlikely(new_point_start_time == new_point_end_time)) {
512
-#ifdef NETDATA_INTERNAL_CHECKS
513
- error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point start time %ld, end time %ld, that are both equal", rd->rrdset->name, rd->name, new_point_start_time, new_point_end_time);
514
-#endif
539
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",
540
+ rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time);
541
+
542
new_point_start_time = new_point_end_time - rd->update_every;
543
}
544
518
- if(unlikely(new_point_start_time < last_point_start_time && new_point_end_time < last_point_end_time)) {
519
-#ifdef NETDATA_INTERNAL_CHECKS
520
- error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point start time %ld, end time %ld, before the last point start time %ld, end time %ld", rd->rrdset->name, rd->name, new_point_start_time, new_point_end_time, last_point_start_time, last_point_end_time);
521
-#endif
522
- new_point_value = last_point_value;
523
- new_point_flags = last_point_flags;
524
- new_point_start_time = last_point_start_time;
525
- new_point_end_time = last_point_end_time;
545
+ if(unlikely(new_point_start_time < last1_point_start_time && new_point_end_time < last1_point_end_time)) {
546
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, before the last point start time %ld, end time %ld",
547
+ rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time,
548
+ last1_point_start_time,
549
+ last1_point_end_time);
550
+
551
+ new_point_value = last1_point_value;
552
+ new_point_flags = last1_point_flags;
553
+ new_point_start_time = last1_point_start_time;
554
+ new_point_end_time = last1_point_end_time;
555
}
556
528
- if(unlikely(new_point_end_time < last_point_end_time)) {
529
-#ifdef NETDATA_INTERNAL_CHECKS
530
- error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point end time %ld, before the last point end time %ld", rd->rrdset->name, rd->name, new_point_end_time, last_point_end_time);
531
-#endif
532
- new_point_value = last_point_value;
533
- new_point_flags = last_point_flags;
534
- new_point_start_time = last_point_start_time;
535
- new_point_end_time = last_point_end_time;
557
+ if(unlikely(new_point_end_time < last1_point_end_time)) {
558
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu end time %ld, before the last point end time %ld",
559
+ rd->rrdset->name, rd->name, db_points_read, new_point_end_time,
560
+ last1_point_end_time);
561
+
562
+ new_point_value = last1_point_value;
563
+ new_point_flags = last1_point_flags;
564
+ new_point_start_time = last1_point_start_time;
565
+ new_point_end_time = last1_point_end_time;
566
}
567
568
if(unlikely(new_point_end_time < now)) {
539
-#ifdef NETDATA_INTERNAL_CHECKS
540
- error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point %ld to %ld, before now (now = %ld, after_wanted = %ld, before_wanted = %ld, dt = %ld)", rd->rrdset->name, rd->name, new_point_start_time, new_point_end_time, now, after_wanted, before_wanted, dt);
541
-#endif
569
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before now (now = %ld, after_wanted = %ld, before_wanted = %ld, dt = %ld)",
570
+ rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time,
571
+ now, after_wanted, before_wanted, query_granularity);
572
+
573
new_point_end_time = now;
574
}
575
}
576
else {
577
new_point_value = NAN;
578
new_point_flags = SN_EMPTY_SLOT;
548
- new_point_start_time = last_point_end_time;
579
+ new_point_start_time = last1_point_end_time;
580
new_point_end_time = now;
581
}
582
583
+ // the inner loop
584
+ // we have 3 points in memory: last, new, next
585
+ // we select the one to use based on their timestamps
586
+
587
size_t iterations = 0;
553
- for ( ; now <= new_point_end_time && points_added < points_wanted; now += dt, iterations++) {
588
+ for ( ; now <= new_point_end_time && points_added < points_wanted; now += query_granularity, iterations++) {
589
+
590
NETDATA_DOUBLE current_point_value;
591
SN_FLAGS current_point_flags;
592
size_t current_point_anomaly;
@@ -559,17 +595,17 @@ static inline void rrd2rrdr_do_dimension(
595
596
if(likely(now > new_point_start_time)) {
597
// it is time for our NEW point to be used
562
- current_point_value = new_point_value;
598
+ current_point_value = interpolate ? interpolate_value(new_point_value, last1_point_value, last1_point_end_time, new_point_start_time, now, new_point_end_time) : new_point_value;
599
current_point_flags = new_point_flags;
600
current_point_anomaly = new_point_anomaly;
601
//current_point_start_time = new_point_start_time;
602
//current_point_end_time = new_point_end_time;
603
}
568
- else if(likely(now <= last_point_end_time)) {
604
+ else if(likely(now <= last1_point_end_time)) {
605
// our LAST point is still valid
570
- current_point_value = last_point_value;
571
- current_point_flags = last_point_flags;
572
- current_point_anomaly = last_point_anomaly;
606
+ current_point_value = interpolate ? interpolate_value(last1_point_value, last2_point_value, last2_point_end_time, last1_point_start_time, now, last1_point_end_time) : last1_point_value;
607
+ current_point_flags = last1_point_flags;
608
+ current_point_anomaly = last1_point_anomaly;
609
//current_point_start_time = last_point_start_time;
610
//current_point_end_time = last_point_end_time;
611
}
@@ -647,7 +683,7 @@ static inline void rrd2rrdr_do_dimension(
683
// but the main loop will increase it,
684
// so, let's undo the last iteration of this loop
685
if(iterations)
650
- now -= dt;
686
+ now -= query_granularity;
687
}
688
rd->state->query_ops.finalize(&handle);
689
@@ -657,11 +693,12 @@ static inline void rrd2rrdr_do_dimension(
693
r->min = min;
694
r->max = max;
695
r->before = max_date;
660
- r->after = min_date - (r->group - 1) * dt;
696
+ r->after = min_date - (r->group - 1) * query_granularity;
697
rrdr_done(r, rrdr_line);
698
663
- if(unlikely(points_wanted != points_added))
664
- error("QUERY: INTERNAL ERROR: query on %s/%s requested %zu points, but RRDR added %zu.", r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added);
699
+ internal_error(points_wanted != points_added,
700
+ "QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).",
701
+ r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, db_points_read);
702
}
703
704
// ----------------------------------------------------------------------------
@@ -670,7 +707,7 @@ static inline void rrd2rrdr_do_dimension(
707
#ifdef NETDATA_INTERNAL_CHECKS
708
static void rrd2rrdr_log_request_response_metadata(RRDR *r
709
, RRDR_GROUPING group_method
673
- , int aligned
710
+ , bool aligned
711
, long group
712
, long resampling_time
713
, long resampling_group
@@ -779,30 +816,17 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before, int up
816
absolute_period_requested = 1;
817
818
// check if the parameters are flipped
782
- if(after_requested >= before_requested) {
819
+ if(after_requested > before_requested) {
820
long long t = before_requested;
821
before_requested = after_requested;
822
after_requested = t;
823
}
824
788
- // we need to make sure that the query is aligned
789
- // with the database update every, otherwise when the user
790
- // requests just 1 point for the entire duration, it may not
791
- // be created (the last 1 point may be misaligned with the
792
- // query).
793
- if(before_requested % update_every)
794
- before_requested += update_every - (before_requested % update_every);
795
-
796
- if(after_requested % update_every)
797
- after_requested -= after_requested % update_every;
798
-
825
// if the query requests future data
826
// shift the query back to be in the present time
827
// (this may also happen because of the rules above)
828
if(before_requested > now) {
829
long long delta = before_requested - now;
804
- if(delta % update_every)
805
- delta += update_every - (delta % update_every);
830
before_requested -= delta;
831
after_requested -= delta;
832
}
@@ -813,7 +837,23 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before, int up
837
return absolute_period_requested;
838
}
839
816
-static RRDR *rrd2rrdr_do_chart(
840
+// #define DEBUG_QUERY_LOGIC 1
841
+
842
+#ifdef DEBUG_QUERY_LOGIC
843
+#define query_debug_log_init() BUFFER *debug_log = buffer_create(1000)
844
+#define query_debug_log(args...) buffer_sprintf(debug_log, ##args)
845
+#define query_debug_log_fin() { \
846
+ info("QUERY: chart '%s', after:%lld, before:%lld, points:%ld, res:%ld - wanted => after:%lld, before:%lld, points:%ld, group:%ld, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", st->name, after_requested, before_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
847
+ buffer_free(debug_log); \
848
+ debug_log = NULL; \
849
+ }
850
+#else
851
+#define query_debug_log_init() debug_dummy()
852
+#define query_debug_log(args...) debug_dummy()
853
+#define query_debug_log_fin() debug_dummy()
854
+#endif
855
+
856
+RRDR *rrd2rrdr(
857
ONEWAYALLOC *owa
858
, RRDSET *st
859
, long points_requested
@@ -823,155 +863,216 @@ static RRDR *rrd2rrdr_do_chart(
863
, long resampling_time_requested
864
, RRDR_OPTIONS options
865
, const char *dimensions
826
- , int update_every
827
- , time_t first_entry_t
828
- , time_t last_entry_t
829
- , int absolute_period_requested
866
, struct context_param *context_param_list
867
, const char *group_options
868
, int timeout
869
) {
834
- UNUSED(last_entry_t);
870
+ // RULES
871
+ // points_requested = 0
872
+ // the user wants all the natural points the database has
873
+ //
874
+ // after_requested = 0
875
+ // the user wants to start the query from the oldest point in our database
876
+ //
877
+ // before_requested = 0
878
+ // the user wants the query to end to the latest point in our database
879
+ //
880
+ // when natural points are wanted, the query has to be aligned to the update_every
881
+ // of the database
882
+
883
+ long points_wanted = points_requested;
884
+ long long after_wanted = after_requested;
885
+ long long before_wanted = before_requested;
886
+ int update_every = st->update_every;
887
+
888
+ bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
889
+ bool automatic_natural_points = (points_wanted == 0);
890
+ bool relative_period_requested = false;
891
+ bool natural_points = (options & RRDR_OPTION_NATURAL_POINTS) || automatic_natural_points;
892
+
893
+ query_debug_log_init();
894
+
895
+ // make sure points_wanted is positive
896
+ if(points_wanted < 0) {
897
+ points_wanted = -points_wanted;
898
+ query_debug_log(":-points_wanted %ld", points_wanted);
899
+ }
900
836
- int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
837
- RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
901
+ if(ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
902
+ relative_period_requested = true;
903
+ natural_points = true;
904
+ options |= RRDR_OPTION_NATURAL_POINTS;
905
+ query_debug_log(":relative+natural");
906
+ }
907
839
- // the duration of the chart
840
- time_t duration = before_requested - after_requested;
841
- long available_points = duration / update_every;
908
+ // this is the update_every of the query
909
+ // it may be different to the update_every of the database
910
+ time_t query_granularity = (natural_points)?update_every:1;
911
+ query_debug_log(":query_granularity %ld", query_granularity);
912
843
- if(duration <= 0 || available_points <= 0)
844
- return NULL;
913
+ if(after_wanted == 0 || before_wanted == 0) {
914
+ // for non-context queries we have to find the duration of the database
915
+ // for context queries we will assume 600 seconds duration
916
846
- if(unlikely(points_requested > available_points))
847
- points_requested = available_points;
917
+ if(!context_param_list) {
918
+ relative_period_requested = true;
919
849
- // calculate the desired grouping of source data points
850
- long group = available_points / points_requested;
851
- if(unlikely(group <= 0)) group = 1;
920
+ rrdset_rdlock(st);
921
+ time_t first_entry_t = rrdset_first_entry_t_nolock(st);
922
+ time_t last_entry_t = rrdset_last_entry_t_nolock(st);
923
+ rrdset_unlock(st);
924
853
- // round "group" to the closest integer
854
- if(unlikely(available_points % points_requested > points_requested / 2))
855
- group++;
925
+ query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
926
857
- // resampling_time_requested enforces a certain grouping multiple
858
- NETDATA_DOUBLE resampling_divisor = 1.0;
859
- long resampling_group = 1;
860
- if(unlikely(resampling_time_requested > update_every)) {
861
- if (unlikely(resampling_time_requested > duration)) {
862
- // group_time is above the available duration
863
-
864
- #ifdef NETDATA_INTERNAL_CHECKS
865
- info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, resampling_time_requested, duration);
866
- #endif
867
-
868
- after_requested = before_requested - resampling_time_requested;
869
- duration = before_requested - after_requested;
870
- available_points = duration / update_every;
871
- group = available_points / points_requested;
872
- }
927
+ if (after_wanted == 0) {
928
+ after_wanted = first_entry_t;
929
+ query_debug_log(":zero after_wanted %lld", after_wanted);
930
+ }
931
874
- // if the duration is not aligned to resampling time
875
- // extend the duration to the past, to avoid a gap at the chart
876
- // only when the missing duration is above 1/10th of a point
877
- if(duration % resampling_time_requested) {
878
- time_t delta = duration % resampling_time_requested;
879
- if(delta > resampling_time_requested / 10) {
880
- after_requested -= resampling_time_requested - delta;
881
- duration = before_requested - after_requested;
882
- available_points = duration / update_every;
883
- group = available_points / points_requested;
932
+ if (before_wanted == 0) {
933
+ before_wanted = last_entry_t;
934
+ query_debug_log(":zero before_wanted %lld", before_wanted);
935
+ }
936
+
937
+ if(points_wanted == 0) {
938
+ points_wanted = (last_entry_t - first_entry_t) / update_every;
939
+ query_debug_log(":zero points_wanted %ld", points_wanted);
940
}
941
}
942
887
- // the points we should group to satisfy gtime
888
- resampling_group = resampling_time_requested / update_every;
889
- if(unlikely(resampling_time_requested % update_every)) {
890
- #ifdef NETDATA_INTERNAL_CHECKS
891
- info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every);
892
- #endif
943
+ // if they are still zero, assume 600
944
894
- resampling_group++;
945
+ if(after_wanted == 0) {
946
+ after_wanted = -600;
947
+ query_debug_log(":zero600 after_wanted %lld", after_wanted);
948
}
949
897
- // adapt group according to resampling_group
898
- if(unlikely(group < resampling_group)) group = resampling_group; // do not allow grouping below the desired one
899
- if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
950
+ if(points_wanted == 0) {
951
+ points_wanted = 600;
952
+ query_debug_log(":zero600 points_wanted %ld", points_wanted);
953
+ }
954
+ }
955
+
956
+ // convert our before_wanted and after_wanted to absolute
957
+ rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, (int)query_granularity, points_wanted);
958
+ query_debug_log(":relative2absolute after %lld, before %lld", after_wanted, before_wanted);
959
+
960
+ // align before_wanted and after_wanted to query_granularity
961
+ if (before_wanted % query_granularity) {
962
+ before_wanted -= before_wanted % query_granularity;
963
+ query_debug_log(":granularity align before_wanted %lld", before_wanted);
964
+ }
965
+
966
+ if (after_wanted % query_granularity) {
967
+ after_wanted -= after_wanted % query_granularity;
968
+ query_debug_log(":granularity align after_wanted %lld", after_wanted);
969
+ }
970
+
971
+ // automatic_natural_points is set when the user wants all the points available in the database
972
+ if(automatic_natural_points) {
973
+ points_wanted = (before_wanted - after_wanted + 1) / query_granularity;
974
+ query_debug_log(":auto natural points_wanted %ld", points_wanted);
975
+ }
976
+
977
+ time_t duration = before_wanted - after_wanted;
978
+
979
+ // if the resampling time is too big, extend the duration to the past
980
+ if (unlikely(resampling_time_requested > duration)) {
981
+ after_wanted = before_wanted - resampling_time_requested;
982
+ duration = before_wanted - after_wanted;
983
+ query_debug_log(":resampling after_wanted %lld", after_wanted);
984
+ }
985
901
- //resampling_divisor = group / resampling_group;
902
- resampling_divisor = (NETDATA_DOUBLE)(group * update_every) / (NETDATA_DOUBLE)resampling_time_requested;
986
+ // if the duration is not aligned to resampling time
987
+ // extend the duration to the past, to avoid a gap at the chart
988
+ // only when the missing duration is above 1/10th of a point
989
+ if(resampling_time_requested > query_granularity && duration % resampling_time_requested) {
990
+ time_t delta = duration % resampling_time_requested;
991
+ if(delta > resampling_time_requested / 10) {
992
+ after_wanted -= resampling_time_requested - delta;
993
+ duration = before_wanted - after_wanted;
994
+ query_debug_log(":resampling2 after_wanted %lld", after_wanted);
995
+ }
996
}
997
905
- // now that we have group,
906
- // align the requested timeframe to fit it.
998
+ // the available points of the query
999
+ long points_available = (duration + 1) / query_granularity;
1000
+ query_debug_log(":points_available %ld", points_available);
1001
908
- if(aligned) {
909
- // alignment has been requested, so align the values
910
- before_requested -= before_requested % (group * update_every);
911
- after_requested -= after_requested % (group * update_every);
1002
+ if(points_wanted > points_available) {
1003
+ points_wanted = points_available;
1004
+ query_debug_log(":max points_wanted %ld", points_wanted);
1005
}
1006
914
- // we align the request on requested_before
915
- time_t before_wanted = before_requested;
1007
+ // calculate the desired grouping of source data points
1008
+ long group = points_available / points_wanted;
1009
+ if(group <= 0) group = 1;
1010
917
- // we need to estimate the number of points, for having
918
- // an integer number of values per point
919
- long points_wanted = (before_wanted - after_requested) / (update_every * group);
1011
+ // round "group" to the closest integer
1012
+ if(points_available % points_wanted > points_wanted / 2)
1013
+ group++;
1014
921
- time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;
922
- if(unlikely(after_wanted < first_entry_t)) {
923
- // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning
924
- points_wanted = (before_wanted - first_entry_t) / group;
1015
+ query_debug_log(":group %ld", group);
1016
926
- // recalculate after wanted with the new number of points
927
- after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;
1017
+ // resampling_time_requested enforces a certain grouping multiple
1018
+ NETDATA_DOUBLE resampling_divisor = 1.0;
1019
+ long resampling_group = 1;
1020
+ if(unlikely(resampling_time_requested > query_granularity)) {
1021
+ // the points we should group to satisfy gtime
1022
+ resampling_group = resampling_time_requested / query_granularity;
1023
+ if(unlikely(resampling_time_requested % query_granularity))
1024
+ resampling_group++;
1025
929
- if(unlikely(after_wanted < first_entry_t)) {
930
- #ifdef NETDATA_INTERNAL_CHECKS
931
- error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name);
932
- #endif
1026
+ query_debug_log(":resampling group %ld", resampling_group);
1027
934
- after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every );
1028
+ // adapt group according to resampling_group
1029
+ if(unlikely(group < resampling_group)) {
1030
+ group = resampling_group; // do not allow grouping below the desired one
1031
+ query_debug_log(":group less res %ld", group);
1032
+ }
1033
+ if(unlikely(group % resampling_group)) {
1034
+ group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
1035
+ query_debug_log(":group mod res %ld", group);
1036
}
936
- }
1037
938
- // check if they are reversed
939
- if(unlikely(after_wanted > before_wanted)) {
940
- #ifdef NETDATA_INTERNAL_CHECKS
941
- error("INTERNAL ERROR: rrd2rrdr() on %s, reversed wanted after/before", st->name);
942
- #endif
943
- time_t tmp = before_wanted;
944
- before_wanted = after_wanted;
945
- after_wanted = tmp;
1038
+ // resampling_divisor = group / resampling_group;
1039
+ resampling_divisor = (NETDATA_DOUBLE)(group * query_granularity) / (NETDATA_DOUBLE)resampling_time_requested;
1040
+ query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor);
1041
}
1042
948
- // recalculate points_wanted using the final time-frame
949
- points_wanted = (before_wanted - after_wanted) / update_every / group + 1;
950
- if(unlikely(points_wanted < 0)) {
951
- #ifdef NETDATA_INTERNAL_CHECKS
952
- error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted);
953
- #endif
954
- points_wanted = 0;
1043
+ // now that we have group, align the requested timeframe to fit it.
1044
+ if(aligned && before_wanted % (group * query_granularity)) {
1045
+ // alignment has been requested, so align the end timestamp
1046
+ before_wanted += (group * query_granularity) - before_wanted % (group * query_granularity);
1047
+ query_debug_log(":align before_wanted %lld", before_wanted);
1048
}
1049
957
-#ifdef NETDATA_INTERNAL_CHECKS
1050
+ after_wanted = before_wanted - (points_wanted * group * query_granularity) + query_granularity;
1051
+ query_debug_log(":final after_wanted %lld", after_wanted);
1052
+
1053
duration = before_wanted - after_wanted;
1054
+ query_debug_log(":final duration %ld", duration);
1055
960
- if(after_wanted < first_entry_t)
961
- error("INTERNAL CHECK: after_wanted %u is too small, minimum %u", (uint32_t)after_wanted, (uint32_t)first_entry_t);
1056
+ // check the context query based on the starting time of the query
1057
+ if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
1058
+ rebuild_context_param_list(owa, context_param_list, after_wanted);
1059
+ st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
1060
963
- if(before_wanted < first_entry_t)
964
- error("INTERNAL CHECK: before_wanted %u is too small, minimum %u", (uint32_t)before_wanted, (uint32_t)first_entry_t);
1061
+ if(unlikely(!st))
1062
+ return NULL;
1063
+ }
1064
966
- if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1)
967
- error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1);
1065
+ internal_error(points_wanted != duration / (query_granularity * group) + 1,
1066
+ "QUERY: points_wanted %ld is not points %ld",
1067
+ points_wanted, duration / (query_granularity * group) + 1);
1068
969
- if(group < resampling_group)
970
- error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group);
1069
+ internal_error(group < resampling_group,
1070
+ "QUERY: group %ld is less than the desired group points %ld",
1071
+ group, resampling_group);
1072
972
- if(group > resampling_group && group % resampling_group)
973
- error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, resampling_group);
974
-#endif
1073
+ internal_error(group > resampling_group && group % resampling_group,
1074
+ "QUERY: group %ld is not a multiple of the desired group points %ld",
1075
+ group, resampling_group);
1076
1077
// -------------------------------------------------------------------------
1078
// initialize our result set
@@ -979,23 +1080,21 @@ static RRDR *rrd2rrdr_do_chart(
1080
1081
RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);
1082
if(unlikely(!r)) {
982
- #ifdef NETDATA_INTERNAL_CHECKS
983
- error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
984
- #endif
1083
+ internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld",
1084
+ st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
1085
return NULL;
1086
}
1087
1088
if(unlikely(!r->d || !points_wanted)) {
989
- #ifdef NETDATA_INTERNAL_CHECKS
990
- error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
991
- #endif
1089
+ internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld",
1090
+ st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
1091
return r;
1092
}
1093
995
- if(unlikely(absolute_period_requested == 1))
996
- r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
997
- else
1094
+ if(relative_period_requested)
1095
r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1096
+ else
1097
+ r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1098
1099
// find how many dimensions we have
1100
long dimensions_count = r->d;
@@ -1004,7 +1103,7 @@ static RRDR *rrd2rrdr_do_chart(
1103
// initialize RRDR
1104
1105
r->group = group;
1007
- r->update_every = (int)group * update_every;
1106
+ r->update_every = (int)(group * query_granularity);
1107
r->before = before_wanted;
1108
r->after = after_wanted;
1109
r->internal.points_wanted = points_wanted;
@@ -1030,19 +1129,22 @@ static RRDR *rrd2rrdr_do_chart(
1129
rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
1130
1131
1132
+ query_debug_log_fin();
1133
+
1134
// -------------------------------------------------------------------------
1135
// do the work for each dimension
1136
1137
time_t max_after = 0, min_before = 0;
1138
long max_rows = 0;
1139
1140
+ RRDDIM *first_rd = context_param_list ? context_param_list->rd : st->dimensions;
1141
RRDDIM *rd;
1142
long c, dimensions_used = 0, dimensions_nonzero = 0;
1143
struct timeval query_start_time;
1144
struct timeval query_current_time;
1043
- if (timeout)
1044
- now_realtime_timeval(&query_start_time);
1045
- for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
1145
+ if (timeout) now_realtime_timeval(&query_start_time);
1146
+
1147
+ for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
1148
1149
// if we need a percentage, we need to calculate all dimensions
1150
if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) {
@@ -1069,26 +1171,23 @@ static RRDR *rrd2rrdr_do_chart(
1171
}
1172
else {
1173
if(r->after != max_after) {
1072
- #ifdef NETDATA_INTERNAL_CHECKS
1073
- error("INTERNAL ERROR: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
1074
- st->name, (size_t)max_after, rd->name, (size_t)r->after);
1075
- #endif
1174
+ internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
1175
+ st->name, (size_t)max_after, rd->name, (size_t)r->after);
1176
+
1177
r->after = (r->after > max_after) ? r->after : max_after;
1178
}
1179
1180
if(r->before != min_before) {
1080
- #ifdef NETDATA_INTERNAL_CHECKS
1081
- error("INTERNAL ERROR: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
1082
- st->name, (size_t)min_before, rd->name, (size_t)r->before);
1083
- #endif
1181
+ internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
1182
+ st->name, (size_t)min_before, rd->name, (size_t)r->before);
1183
+
1184
r->before = (r->before < min_before) ? r->before : min_before;
1185
}
1186
1187
if(r->rows != max_rows) {
1088
- #ifdef NETDATA_INTERNAL_CHECKS
1089
- error("INTERNAL ERROR: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
1090
- st->name, (size_t)max_rows, rd->name, (size_t)r->rows);
1091
- #endif
1188
+ internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
1189
+ st->name, (size_t)max_rows, rd->name, (size_t)r->rows);
1190
+
1191
r->rows = (r->rows > max_rows) ? r->rows : max_rows;
1192
}
1193
}
@@ -1102,32 +1201,50 @@ static RRDR *rrd2rrdr_do_chart(
1201
}
1202
}
1203
1105
- #ifdef NETDATA_INTERNAL_CHECKS
1204
+#ifdef NETDATA_INTERNAL_CHECKS
1205
if (dimensions_used) {
1206
if(r->internal.log)
1108
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log);
1207
+ rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted,
1208
+ after_wanted, before_wanted,
1209
+ before_wanted,
1210
+ points_wanted, points_wanted, /*after_slot, before_slot,*/ r->internal.log);
1211
1212
if(r->rows != points_wanted)
1111
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'");
1213
+ rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted,
1214
+ after_wanted, before_wanted,
1215
+ before_wanted,
1216
+ points_wanted, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'");
1217
1113
- if(aligned && (r->before % group) != 0)
1114
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required");
1218
+ if(aligned && (r->before % (group * query_granularity)) != 0)
1219
+ rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted,
1220
+ after_wanted, before_wanted,
1221
+ before_wanted,
1222
+ points_wanted, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required");
1223
1224
// 'after' should not be aligned, since we start inside the first group
1225
//if(aligned && (r->after % group) != 0)
1226
// rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
1227
1120
- if(r->before != before_requested)
1121
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'");
1228
+ if(r->before != before_wanted)
1229
+ rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted,
1230
+ after_wanted, before_wanted,
1231
+ before_wanted,
1232
+ points_wanted, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'");
1233
1234
if(r->before != before_wanted)
1124
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'");
1235
+ rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted,
1236
+ after_wanted, before_wanted,
1237
+ before_wanted,
1238
+ points_wanted, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'");
1239
1240
// reported 'after' varies, depending on group
1241
if(r->after != after_wanted)
1128
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'");
1242
+ rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted,
1243
+ after_wanted, before_wanted,
1244
+ before_wanted,
1245
+ points_wanted, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'");
1246
}
1130
- #endif
1247
+#endif
1248
1249
// free all resources used by the grouping method
1250
r->internal.grouping_free(r);
@@ -1136,7 +1253,7 @@ static RRDR *rrd2rrdr_do_chart(
1253
if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
1254
// all the dimensions are zero
1255
// mark them as NONZERO to send them all
1139
- for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
1256
+ for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
1257
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
1258
r->od[c] |= RRDR_DIMENSION_NONZERO;
1259
}
@@ -1145,76 +1262,3 @@ static RRDR *rrd2rrdr_do_chart(
1262
rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);
1263
return r;
1264
}
1148
-
1149
-RRDR *rrd2rrdr(
1150
- ONEWAYALLOC *owa
1151
- , RRDSET *st
1152
- , long points_requested
1153
- , long long after_requested
1154
- , long long before_requested
1155
- , RRDR_GROUPING group_method
1156
- , long resampling_time_requested
1157
- , RRDR_OPTIONS options
1158
- , const char *dimensions
1159
- , struct context_param *context_param_list
1160
- , const char *group_options
1161
- , int timeout
1162
-)
1163
-{
1164
- int rrd_update_every = st->update_every;
1165
- int absolute_period_requested;
1166
-
1167
- if(unlikely(points_requested < 0))
1168
- points_requested = -points_requested;
1169
-
1170
- long points_original = points_requested;
1171
- if(unlikely(!points_requested))
1172
- points_requested = (before_requested - after_requested) / rrd_update_every;
1173
-
1174
- if(unlikely(!points_requested))
1175
- points_requested = 1;
1176
-
1177
- time_t first_entry_t;
1178
- time_t last_entry_t;
1179
- if (context_param_list) {
1180
- first_entry_t = context_param_list->first_entry_t;
1181
- last_entry_t = context_param_list->last_entry_t;
1182
- }
1183
- else {
1184
- rrdset_rdlock(st);
1185
- first_entry_t = rrdset_first_entry_t_nolock(st);
1186
- last_entry_t = rrdset_last_entry_t_nolock(st);
1187
- rrdset_unlock(st);
1188
- }
1189
-
1190
- absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested,
1191
- rrd_update_every, points_requested);
1192
-
1193
- if(options & RRDR_OPTION_ALLOW_PAST) {
1194
- if (first_entry_t > after_requested)
1195
- first_entry_t = after_requested;
1196
-
1197
- if (last_entry_t < before_requested)
1198
- last_entry_t = before_requested;
1199
- }
1200
- else {
1201
- if(after_requested < first_entry_t)
1202
- after_requested = first_entry_t;
1203
- }
1204
-
1205
- if(!points_original)
1206
- points_requested = (before_requested - after_requested) / rrd_update_every;
1207
-
1208
- if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
1209
- rebuild_context_param_list(owa, context_param_list, after_requested);
1210
- st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
1211
- if (unlikely(!st))
1212
- return NULL;
1213
- }
1214
-
1215
- return rrd2rrdr_do_chart(owa, st, points_requested,
1216
- after_requested, before_requested,
1217
- group_method, resampling_time_requested, options, dimensions, rrd_update_every,
1218
- first_entry_t, last_entry_t, absolute_period_requested,
1219
- context_param_list, group_options, timeout);
1220
-}
web/api/queries/rrdr.h
+22
-22
@@ -6,27 +6,27 @@
6
#include "libnetdata/libnetdata.h"
7
8
typedef enum rrdr_options {
9
- RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
10
- RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest)
11
- RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing
12
- RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum
13
- RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates
14
- RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates
15
- RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros
16
- RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array
17
- RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs
18
- RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result
19
- RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes
20
- RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total
21
- RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes
22
- RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
23
- RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs
24
- RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
25
- RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response
26
- RRDR_OPTION_ALLOW_PAST = 0x00020000, // The after parameter can extend in the past before the first entry
27
- RRDR_OPTION_ANOMALY_BIT = 0x00040000, // Return the anomaly bit stored in each collected_number
28
- RRDR_OPTION_RETURN_RAW = 0x00080000, // Return raw data for aggregating across multiple nodes
29
- RRDR_OPTION_RETURN_JWAR = 0x00100000, // Return anomaly rates in jsonwrap
9
+ RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
10
+ RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest)
11
+ RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing
12
+ RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum
13
+ RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates
14
+ RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates
15
+ RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros
16
+ RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array
17
+ RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs
18
+ RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result
19
+ RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes
20
+ RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total
21
+ RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes
22
+ RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
23
+ RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs
24
+ RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
25
+ RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response
26
+ RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
27
+ RRDR_OPTION_ANOMALY_BIT = 0x00040000, // Return the anomaly bit stored in each collected_number
28
+ RRDR_OPTION_RETURN_RAW = 0x00080000, // Return raw data for aggregating across multiple nodes
29
+ RRDR_OPTION_RETURN_JWAR = 0x00100000, // Return anomaly rates in jsonwrap
30
31
// internal ones - not to be exposed to the API
32
RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
@@ -116,7 +116,7 @@ extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct con
116
117
extern RRDR *rrd2rrdr(
118
ONEWAYALLOC *owa,
119
- RRDSET *st, long points_requested, long long after_requested, long long before_requested,
119
+ RRDSET *st, long points_wanted, long long after_wanted, long long before_wanted,
120
RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
121
struct context_param *context_param_list, const char *group_options, int timeout);
122
web/api/web_api_v1.c
+2
-1
@@ -38,6 +38,7 @@ static struct {
38
, {"anomaly-bit" , 0 , RRDR_OPTION_ANOMALY_BIT}
39
, {"raw" , 0 , RRDR_OPTION_RETURN_RAW}
40
, {"jw-anomaly-rates" , 0 , RRDR_OPTION_RETURN_JWAR}
41
+ , {"natural-points" , 0 , RRDR_OPTION_NATURAL_POINTS}
42
, {NULL , 0 , 0}
43
};
44
@@ -1326,7 +1327,7 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie
1327
return HTTP_RESP_BACKEND_FETCH_FAILED;
1328
1329
long long baseline_after = 0, baseline_before = 0, after = 0, before = 0, points = 0;
1329
- RRDR_OPTIONS options = RRDR_OPTION_NOT_ALIGNED | RRDR_OPTION_NONZERO | RRDR_OPTION_NULL2ZERO | RRDR_OPTION_ALLOW_PAST;
1330
+ RRDR_OPTIONS options = RRDR_OPTION_NOT_ALIGNED | RRDR_OPTION_NONZERO | RRDR_OPTION_NULL2ZERO;
1331
METRIC_CORRELATIONS_METHOD method = default_metric_correlations_method;
1332
RRDR_GROUPING group = RRDR_GROUPING_AVERAGE;
1333
int timeout = 0;