speedup queries by providing optimization in the main loop (#12811)
Costa Tsaousis committed
May 7, 2022 at 08:38 UTC
5b48a70abde5903f0dc99b19b674b85134ac9537
4 files changed
+37
-10
database/engine/rrdengine.h
+3
@@ -49,6 +49,9 @@ struct rrdeng_query_handle {
49
time_t next_page_time;
50
time_t now;
51
unsigned position;
52
+ storage_number *page;
53
+ usec_t page_end_time;
54
+ uint32_t page_length;
55
};
56
57
typedef enum {
database/engine/rrdengineapi.c
+11
-2
@@ -573,7 +573,10 @@ storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle
573
/* it's the first call */
574
next_page_time = handle->next_page_time * USEC_PER_SEC;
575
} else {
576
- pg_cache_atomic_get_pg_info(descr, &page_end_time, &page_length);
576
+ // pg_cache_atomic_get_pg_info(descr, &page_end_time, &page_length);
577
+ page_end_time = handle->page_end_time;
578
+ page_length = handle->page_length;
579
+ page = handle->page;
580
}
581
position = handle->position + 1;
582
@@ -616,10 +619,16 @@ storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle
619
} else {
620
position = 0;
621
}
622
+
623
+ handle->page_end_time = page_end_time;
624
+ handle->page_length = page_length;
625
+ page = handle->page = descr->pg_cache_descr->page;
626
}
620
- page = descr->pg_cache_descr->page;
627
+
628
+ // page = descr->pg_cache_descr->page;
629
ret = page[position];
630
entries = page_length / sizeof(storage_number);
631
+
632
if (entries > 1) {
633
usec_t dt;
634
web/api/queries/average/average.c
+1
-1
@@ -29,7 +29,7 @@ void grouping_free_average(RRDR *r) {
29
}
30
31
void grouping_add_average(RRDR *r, calculated_number value) {
32
- if(!isnan(value)) {
32
+ if(likely(!isnan(value))) {
33
struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
34
g->sum += value;
35
g->count++;
web/api/queries/query.c
+22
-7
@@ -563,6 +563,13 @@ static inline void do_dimension_fixedstep(
563
size_t db_points_read = 0;
564
time_t db_now = now;
565
time_t first_time_t = rrddim_first_entry_t(rd);
566
+
567
+ // cache the function pointers we need in the loop
568
+ storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time) = rd->state->query_ops.next_metric;
569
+ void (*grouping_add)(struct rrdresult *r, calculated_number value) = r->internal.grouping_add;
570
+ calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) = r->internal.grouping_flush;
571
+ RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;
572
+
573
for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) {
574
// make sure we return data in the proper time range
575
if(unlikely(now > before_wanted)) {
@@ -571,35 +578,42 @@ static inline void do_dimension_fixedstep(
578
#endif
579
break;
580
}
581
+
582
if(unlikely(now < after_wanted)) {
583
#ifdef NETDATA_INTERNAL_CHECKS
584
r->internal.log = "skipped, because attempted to access the db before 'wanted after'";
585
#endif
586
continue;
587
}
588
+
589
// read the value from the database
590
//storage_number n = rd->values[slot];
591
+
592
#ifdef NETDATA_INTERNAL_CHECKS
593
struct mem_query_handle* mem_handle = (struct mem_query_handle*)handle.handle;
584
- if ((rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) &&
594
+ if ((rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) &&
595
(rrdset_time2slot(st, now) != (long unsigned)(mem_handle->slot))) {
596
error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)mem_handle->slot, rrdset_time2slot(st, now));
597
}
598
#endif
599
+
600
db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it
601
storage_number n;
591
- if (rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t)
602
+ if (unlikely(rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t))
603
n = SN_EMPTY_SLOT;
604
else
594
- n = rd->state->query_ops.next_metric(&handle, &db_now);
605
+ n = next_metric(&handle, &db_now);
606
+
607
if(unlikely(db_now > before_wanted)) {
608
#ifdef NETDATA_INTERNAL_CHECKS
609
r->internal.log = "stopped, because attempted to access the db after 'wanted before'";
610
#endif
611
break;
612
}
613
+
614
for ( ; now <= db_now ; now += dt) {
615
calculated_number value = NAN;
616
+
617
if(likely(now >= db_now && does_storage_number_exist(n))) {
618
#if defined(NETDATA_INTERNAL_CHECKS) && defined(ENABLE_DBENGINE)
619
struct rrdeng_query_handle* rrd_handle = (struct rrdeng_query_handle*)handle.handle;
@@ -621,18 +635,19 @@ static inline void do_dimension_fixedstep(
635
}
636
637
// add this value for grouping
624
- r->internal.grouping_add(r, value);
638
+ grouping_add(r, value);
639
values_in_group++;
640
db_points_read++;
641
642
if(unlikely(values_in_group == group_size)) {
643
rrdr_line = rrdr_line_init(r, now, rrdr_line);
644
+ size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
645
646
if(unlikely(!min_date)) min_date = now;
647
max_date = now;
648
649
// find the place to store our values
635
- RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr];
650
+ RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];
651
652
// update the dimension options
653
if(likely(values_in_group_non_zero))
@@ -642,8 +657,8 @@ static inline void do_dimension_fixedstep(
657
*rrdr_value_options_ptr = group_value_flags;
658
659
// store the value
645
- calculated_number value = r->internal.grouping_flush(r, rrdr_value_options_ptr);
646
- r->v[rrdr_line * r->d + dim_id_in_rrdr] = value;
660
+ value = grouping_flush(r, rrdr_value_options_ptr);
661
+ r->v[rrdr_o_v_index] = value;
662
663
if(likely(points_added || dim_id_in_rrdr)) {
664
// find the min/max across all dimensions