85b61ab65 · cryptotaxi247/netdata-1

@@ -564,21 +564,33 @@ static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t aft

564 // query ops

565

566 typedef struct query_point {

567 + time_t end_time;

568 + time_t start_time;

569 NETDATA_DOUBLE value;

568 - SN_FLAGS flags;

570 size_t anomaly;

570 - time_t start_time;

571 - time_t end_time;

571 + SN_FLAGS flags;

572 +#ifdef NETDATA_INTERNAL_CHECKS

573 + size_t id;

574 +#endif

575 } QUERY_POINT;

576

577 QUERY_POINT QUERY_POINT_EMPTY = {

578 + .end_time = 0,

579 + .start_time = 0,

580 .value = NAN,

576 - .flags = SN_EMPTY_SLOT,

581 .anomaly = 0,

578 - .start_time = 0,

579 - .end_time = 0

582 + .flags = SN_EMPTY_SLOT,

583 +#ifdef NETDATA_INTERNAL_CHECKS

584 + .id = 0,

585 +#endif

586 };

587

588 +#ifdef NETDATA_INTERNAL_CHECKS

589 +#define query_point_set_id(point, point_id) (point).id = point_id

590 +#else

591 +#define query_point_set_id(point, point_id) debug_dummy()

592 +#endif

593 +

594 typedef struct query_plan_entry {

595 size_t tier;

596 time_t after;

@@ -859,13 +871,18 @@ static inline void rrd2rrdr_do_dimension(

871 QUERY_POINT last1_point = QUERY_POINT_EMPTY;

872 QUERY_POINT new_point = QUERY_POINT_EMPTY;

873

874 + time_t now_start_time = after_wanted - ops.query_granularity;

875 + time_t now_end_time = after_wanted + ops.view_update_every - ops.query_granularity;

876 +

877 // The main loop, based on the query granularity we need

863 - for(time_t now = after_wanted + ops.view_update_every - ops.query_granularity; (long)points_added < points_wanted ; now += ops.view_update_every) {

878 + for( ; (long)points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {

879 +

880 + if(query_plan_should_switch_plan(ops, now_end_time))

881 + query_planer_next_plan(&ops, now_end_time, new_point.end_time);

882 +

883 + // read all the points of the db, prior to the time we need (now_end_time)

884

865 - if(query_plan_should_switch_plan(ops, now))

866 - query_planer_next_plan(&ops, now, new_point.end_time);

885

868 - // real all the points of the db, prior to the time we need (now)

886 size_t count_same_end_time = 0;

887 while(count_same_end_time < 100) {

888 if(likely(count_same_end_time == 0)) {

@@ -880,7 +897,7 @@ static inline void rrd2rrdr_do_dimension(

897 }

898 new_point = QUERY_POINT_EMPTY;

899 new_point.start_time = last1_point.end_time;

883 - new_point.end_time = now;

900 + new_point.end_time = now_end_time;

901 break;

902 }

903

@@ -894,7 +911,9 @@ static inline void rrd2rrdr_do_dimension(

911 new_point.start_time = sp.start_time;

912 new_point.end_time = sp.end_time;

913 new_point.anomaly = sp.count ? sp.anomaly_count * 100 / sp.count : 0;

914 + query_point_set_id(new_point, ops.db_total_points_read);

915

916 + // set the right value to the point we got

917 if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {

918

919 if(unlikely(use_anomaly_bit_as_value))

@@ -927,35 +946,66 @@ static inline void rrd2rrdr_do_dimension(

946 }

947 }

948

949 + // check if the db is giving us zero duration points

950 if(unlikely(new_point.start_time == new_point.end_time)) {

951 internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",

932 - rd->rrdset->name, rd->name, ops.db_total_points_read, new_point.start_time, new_point.end_time);

952 + rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time);

953

954 new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every);

955 }

956

957 + // check if the db is advancing the query

958 if(unlikely(new_point.end_time <= last1_point.end_time)) {

938 - internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point end time %ld, now is %ld",

939 - rd->rrdset->name, rd->name, ops.db_total_points_read, new_point.start_time, new_point.end_time, last1_point.end_time, now);

959 + internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld",

960 + rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time,

961 + last1_point.id, last1_point.end_time, now_start_time, now_end_time);

962

963 count_same_end_time++;

964 continue;

965 }

944 -

966 count_same_end_time = 0;

967

947 - if(new_point.end_time < now)

948 - query_add_point_to_group(r, new_point, ops);

949 - else

968 + // decide how to use this point

969 + if(likely(new_point.end_time < now_end_time)) { // likely to favor tier0

970 + // this db point ends before our now_end_time

971 +

972 + if(likely(new_point.end_time >= now_start_time)) { // likely to favor tier0

973 + // this db point ends after our now_start time

974 +

975 + query_add_point_to_group(r, new_point, ops);

976 + }

977 + else {

978 + // we don't need this db point

979 + // it is totally outside our current time-frame

980 +

981 + // this is desirable for the first point of the query

982 + // because it allows us to interpolate the next point

983 + // at exactly the time we will want

984 +

985 + // we only log if this is not point 1

986 + internal_error(new_point.end_time < after_wanted && new_point.id > 1,

987 + "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",

988 + rd->rrdset->name, rd->name,

989 + new_point.id, new_point.start_time, new_point.end_time,

990 + now_start_time, now_end_time,

991 + after_wanted, before_wanted);

992 + }

993 +

994 + }

995 + else {

996 + // the point ends in the future

997 + // so, we will interpolate it below, at the inner loop

998 break;

999 + }

1000 }

1001

953 - if(count_same_end_time) {

1002 + if(unlikely(count_same_end_time)) {

1003 internal_error(true,

955 - "QUERY: the database does not advance the query, it returned an end time less or equal to %ld, %zu times",

1004 + "QUERY: the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",

1005 last1_point.end_time, count_same_end_time);

1006

958 - new_point.end_time = now;

1007 + if(unlikely(new_point.end_time <= last1_point.end_time))

1008 + new_point.end_time = now_end_time;

1009 }

1010

1011 // the inner loop

@@ -963,18 +1013,35 @@ static inline void rrd2rrdr_do_dimension(

1013 // we select the one to use based on their timestamps

1014

1015 size_t iterations = 0;

966 - for ( ; now <= new_point.end_time && (long)points_added < points_wanted; now += ops.view_update_every, iterations++) {

1016 + for ( ; now_end_time <= new_point.end_time && (long)points_added < points_wanted ;

1017 + now_end_time += ops.view_update_every, iterations++) {

1018 +

1019 + // now_start_time is wrong in this loop

1020 + // but, we don't need it

1021 +

1022 QUERY_POINT current_point;

1023

969 - if(likely(now > new_point.start_time)) {

1024 + if(likely(now_end_time > new_point.start_time)) {

1025 // it is time for our NEW point to be used

1026 current_point = new_point;

972 - query_interpolate_point(current_point, last1_point, now);

1027 + query_interpolate_point(current_point, last1_point, now_end_time);

1028 +

1029 + internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,

1030 + "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"

1031 + " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.",

1032 + rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,

1033 + current_point.id, current_point.start_time, current_point.end_time, now_end_time);

1034 }

974 - else if(likely(now <= last1_point.end_time)) {

1035 + else if(likely(now_end_time <= last1_point.end_time)) {

1036 // our LAST point is still valid

1037 current_point = last1_point;

977 - query_interpolate_point(current_point, last2_point, now);

1038 + query_interpolate_point(current_point, last2_point, now_end_time);

1039 +

1040 + internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,

1041 + "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"

1042 + " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.",

1043 + rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,

1044 + current_point.id, current_point.start_time, current_point.end_time, now_end_time);

1045 }

1046 else {

1047 // a GAP, we don't have a value this time

@@ -983,11 +1050,11 @@ static inline void rrd2rrdr_do_dimension(

1050

1051 query_add_point_to_group(r, current_point, ops);

1052

986 - rrdr_line = rrdr_line_init(r, now, rrdr_line);

1053 + rrdr_line = rrdr_line_init(r, now_end_time, rrdr_line);

1054 size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;

1055

989 - if(unlikely(!min_date)) min_date = now;

990 - max_date = now;

1056 + if(unlikely(!min_date)) min_date = now_end_time;

1057 + max_date = now_end_time;

1058

1059 // find the place to store our values

1060 RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];

@@ -1032,7 +1099,7 @@ static inline void rrd2rrdr_do_dimension(

1099 // but the main loop will increase it too,

1100 // so, let's undo the last iteration of this loop

1101 if(iterations)

1035 - now -= ops.view_update_every;

1102 + now_end_time -= ops.view_update_every;

1103 }

1104 ops.finalize(&ops.handle);

1105

query engine: omit first point if not needed (#13345)