| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "query-internal.h" |
| 4 | |
| 5 | // #define DEBUG_QUERY_LOGIC 1 |
| 6 | |
| 7 | #ifdef DEBUG_QUERY_LOGIC |
| 8 | #define query_debug_log_init() BUFFER *debug_log = buffer_create(1000) |
| 9 | #define query_debug_log(args...) buffer_sprintf(debug_log, ##args) |
| 10 | #define query_debug_log_fin() { \ |
| 11 | netdata_log_info("QUERY: '%s', after:%ld, before:%ld, duration:%ld, points:%zu, res:%ld - wanted => after:%ld, before:%ld, points:%zu, group:%zu, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", qt->id, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \ |
| 12 | buffer_free(debug_log); \ |
| 13 | debug_log = NULL; \ |
| 14 | } |
| 15 | #define query_debug_log_free() do { buffer_free(debug_log); } while(0) |
| 16 | #else |
| 17 | #define query_debug_log_init() debug_dummy() |
| 18 | #define query_debug_log(args...) debug_dummy() |
| 19 | #define query_debug_log_fin() debug_dummy() |
| 20 | #define query_debug_log_free() debug_dummy() |
| 21 | #endif |
| 22 | |
| 23 | bool query_target_calculate_window(QUERY_TARGET *qt) { |
| 24 | if (unlikely(!qt)) return false; |
| 25 | |
| 26 | size_t points_requested = (long)qt->request.points; |
| 27 | time_t after_requested = qt->request.after; |
| 28 | time_t before_requested = qt->request.before; |
| 29 | RRDR_TIME_GROUPING group_method = qt->request.time_group_method; |
| 30 | time_t resampling_time_requested = qt->request.resampling_time; |
| 31 | RRDR_OPTIONS options = qt->window.options; |
| 32 | size_t tier = qt->request.tier; |
| 33 | time_t update_every = qt->db.minimum_latest_update_every_s ? qt->db.minimum_latest_update_every_s : 1; |
| 34 | |
| 35 | // RULES |
| 36 | // points_requested = 0 |
| 37 | // the user wants all the natural points the database has |
| 38 | // |
| 39 | // after_requested = 0 |
| 40 | // the user wants to start the query from the oldest point in our database |
| 41 | // |
| 42 | // before_requested = 0 |
| 43 | // the user wants the query to end to the latest point in our database |
| 44 | // |
| 45 | // when natural points are wanted, the query has to be aligned to the update_every |
| 46 | // of the database |
| 47 | |
| 48 | size_t points_wanted = points_requested; |
| 49 | time_t after_wanted = after_requested; |
| 50 | time_t before_wanted = before_requested; |
| 51 | |
| 52 | bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED); |
| 53 | bool automatic_natural_points = (points_wanted == 0); |
| 54 | bool relative_period_requested = false; |
| 55 | bool natural_points = (options & RRDR_OPTION_NATURAL_POINTS) || automatic_natural_points; |
| 56 | bool before_is_aligned_to_db_end = false; |
| 57 | |
| 58 | query_debug_log_init(); |
| 59 | |
| 60 | if (ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) { |
| 61 | relative_period_requested = true; |
| 62 | natural_points = true; |
| 63 | options |= RRDR_OPTION_NATURAL_POINTS; |
| 64 | query_debug_log(":relative+natural"); |
| 65 | } |
| 66 | |
| 67 | // if the user wants virtual points, make sure we do it |
| 68 | if (options & RRDR_OPTION_VIRTUAL_POINTS) |
| 69 | natural_points = false; |
| 70 | |
| 71 | // set the right flag about natural and virtual points |
| 72 | if (natural_points) { |
| 73 | options |= RRDR_OPTION_NATURAL_POINTS; |
| 74 | |
| 75 | if (options & RRDR_OPTION_VIRTUAL_POINTS) |
| 76 | options &= ~RRDR_OPTION_VIRTUAL_POINTS; |
| 77 | } |
| 78 | else { |
| 79 | options |= RRDR_OPTION_VIRTUAL_POINTS; |
| 80 | |
| 81 | if (options & RRDR_OPTION_NATURAL_POINTS) |
| 82 | options &= ~RRDR_OPTION_NATURAL_POINTS; |
| 83 | } |
| 84 | |
| 85 | if (after_wanted == 0 || before_wanted == 0) { |
| 86 | relative_period_requested = true; |
| 87 | |
| 88 | time_t first_entry_s = qt->db.first_time_s; |
| 89 | time_t last_entry_s = qt->db.last_time_s; |
| 90 | |
| 91 | if (first_entry_s == 0 || last_entry_s == 0) { |
| 92 | internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld)", qt->id, first_entry_s, last_entry_s); |
| 93 | after_wanted = qt->window.after; |
| 94 | before_wanted = qt->window.before; |
| 95 | |
| 96 | if(after_wanted == before_wanted) |
| 97 | after_wanted = before_wanted - update_every; |
| 98 | |
| 99 | if (points_wanted == 0) { |
| 100 | points_wanted = (before_wanted - after_wanted) / update_every; |
| 101 | query_debug_log(":zero points_wanted %zu", points_wanted); |
| 102 | } |
| 103 | } |
| 104 | else { |
| 105 | query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_s, last_entry_s); |
| 106 | |
| 107 | if (after_wanted == 0) { |
| 108 | after_wanted = first_entry_s; |
| 109 | query_debug_log(":zero after_wanted %ld", after_wanted); |
| 110 | } |
| 111 | |
| 112 | if (before_wanted == 0) { |
| 113 | before_wanted = last_entry_s; |
| 114 | before_is_aligned_to_db_end = true; |
| 115 | query_debug_log(":zero before_wanted %ld", before_wanted); |
| 116 | } |
| 117 | |
| 118 | if (points_wanted == 0) { |
| 119 | points_wanted = (last_entry_s - first_entry_s) / update_every; |
| 120 | query_debug_log(":zero points_wanted %zu", points_wanted); |
| 121 | } |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | if (points_wanted == 0) { |
| 126 | points_wanted = 600; |
| 127 | query_debug_log(":zero600 points_wanted %zu", points_wanted); |
| 128 | } |
| 129 | |
| 130 | // convert our before_wanted and after_wanted to absolute |
| 131 | rrdr_relative_window_to_absolute_query(&after_wanted, &before_wanted, NULL, unittest_running); |
| 132 | query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted); |
| 133 | |
| 134 | if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && |
| 135 | tier > 0 && tier < nd_profile.storage_tiers && nd_profile.storage_tiers > 1) { |
| 136 | update_every = query_target_min_update_every_for_tier(qt, tier); |
| 137 | |
| 138 | if (update_every <= 0) update_every = qt->db.minimum_latest_update_every_s; |
| 139 | query_debug_log(":natural update every %ld", update_every); |
| 140 | } |
| 141 | |
| 142 | // this is the update_every of the query |
| 143 | // it may be different to the update_every of the database |
| 144 | time_t query_granularity = (natural_points) ? update_every : 1; |
| 145 | if (query_granularity <= 0) query_granularity = 1; |
| 146 | query_debug_log(":query_granularity %ld", query_granularity); |
| 147 | |
| 148 | // align before_wanted and after_wanted to query_granularity |
| 149 | if (before_wanted % query_granularity) { |
| 150 | before_wanted -= before_wanted % query_granularity; |
| 151 | query_debug_log(":granularity align before_wanted %ld", before_wanted); |
| 152 | } |
| 153 | |
| 154 | if (after_wanted % query_granularity) { |
| 155 | after_wanted -= after_wanted % query_granularity; |
| 156 | query_debug_log(":granularity align after_wanted %ld", after_wanted); |
| 157 | } |
| 158 | |
| 159 | // automatic_natural_points is set when the user wants all the points available in the database |
| 160 | if (automatic_natural_points) { |
| 161 | points_wanted = (before_wanted - after_wanted + 1) / query_granularity; |
| 162 | if (unlikely(points_wanted <= 0)) points_wanted = 1; |
| 163 | query_debug_log(":auto natural points_wanted %zu", points_wanted); |
| 164 | } |
| 165 | |
| 166 | time_t duration = before_wanted - after_wanted; |
| 167 | |
| 168 | // if the resampling time is too big, extend the duration to the past |
| 169 | if (unlikely(resampling_time_requested > duration)) { |
| 170 | after_wanted = before_wanted - resampling_time_requested; |
| 171 | duration = before_wanted - after_wanted; |
| 172 | query_debug_log(":resampling after_wanted %ld", after_wanted); |
| 173 | } |
| 174 | |
| 175 | // if the duration is not aligned to resampling time |
| 176 | // extend the duration to the past, to avoid a gap at the chart |
| 177 | // only when the missing duration is above 1/10th of a point |
| 178 | if (resampling_time_requested > query_granularity && duration % resampling_time_requested) { |
| 179 | time_t delta = duration % resampling_time_requested; |
| 180 | if (delta > resampling_time_requested / 10) { |
| 181 | after_wanted -= resampling_time_requested - delta; |
| 182 | duration = before_wanted - after_wanted; |
| 183 | query_debug_log(":resampling2 after_wanted %ld", after_wanted); |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | // the available points of the query |
| 188 | size_t points_available = (duration + 1) / query_granularity; |
| 189 | if (unlikely(points_available <= 0)) points_available = 1; |
| 190 | query_debug_log(":points_available %zu", points_available); |
| 191 | |
| 192 | if (points_wanted > points_available) { |
| 193 | points_wanted = points_available; |
| 194 | query_debug_log(":max points_wanted %zu", points_wanted); |
| 195 | } |
| 196 | |
| 197 | if(points_wanted > 86400 && !unittest_running) { |
| 198 | points_wanted = 86400; |
| 199 | query_debug_log(":absolute max points_wanted %zu", points_wanted); |
| 200 | } |
| 201 | |
| 202 | // calculate the desired grouping of source data points |
| 203 | size_t group = points_available / points_wanted; |
| 204 | if (group == 0) group = 1; |
| 205 | |
| 206 | // round "group" to the closest integer |
| 207 | if (points_available % points_wanted > points_wanted / 2) |
| 208 | group++; |
| 209 | |
| 210 | query_debug_log(":group %zu", group); |
| 211 | |
| 212 | if (points_wanted * group * query_granularity < (size_t)duration) { |
| 213 | // the grouping we are going to do, is not enough |
| 214 | // to cover the entire duration requested, so |
| 215 | // we have to change the number of points, to make sure we will |
| 216 | // respect the timeframe as closely as possibly |
| 217 | |
| 218 | // let's see how many points are the optimal |
| 219 | points_wanted = points_available / group; |
| 220 | |
| 221 | if (points_wanted * group < points_available) |
| 222 | points_wanted++; |
| 223 | |
| 224 | if (unlikely(points_wanted == 0)) |
| 225 | points_wanted = 1; |
| 226 | |
| 227 | query_debug_log(":optimal points %zu", points_wanted); |
| 228 | } |
| 229 | |
| 230 | // resampling_time_requested enforces a certain grouping multiple |
| 231 | NETDATA_DOUBLE resampling_divisor = 1.0; |
| 232 | size_t resampling_group = 1; |
| 233 | if (unlikely(resampling_time_requested > query_granularity)) { |
| 234 | // the points we should group to satisfy gtime |
| 235 | resampling_group = resampling_time_requested / query_granularity; |
| 236 | if (unlikely(resampling_time_requested % query_granularity)) |
| 237 | resampling_group++; |
| 238 | |
| 239 | query_debug_log(":resampling group %zu", resampling_group); |
| 240 | |
| 241 | // adapt group according to resampling_group |
| 242 | if (unlikely(group < resampling_group)) { |
| 243 | group = resampling_group; // do not allow grouping below the desired one |
| 244 | query_debug_log(":group less res %zu", group); |
| 245 | } |
| 246 | if (unlikely(group % resampling_group)) { |
| 247 | group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group |
| 248 | query_debug_log(":group mod res %zu", group); |
| 249 | } |
| 250 | |
| 251 | // resampling_divisor = group / resampling_group; |
| 252 | resampling_divisor = (NETDATA_DOUBLE) (group * query_granularity) / (NETDATA_DOUBLE) resampling_time_requested; |
| 253 | query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor); |
| 254 | } |
| 255 | |
| 256 | // now that we have group, align the requested timeframe to fit it. |
| 257 | if (aligned && before_wanted % (group * query_granularity)) { |
| 258 | if (before_is_aligned_to_db_end) |
| 259 | before_wanted -= before_wanted % (time_t)(group * query_granularity); |
| 260 | else |
| 261 | before_wanted += (time_t)(group * query_granularity) - before_wanted % (time_t)(group * query_granularity); |
| 262 | query_debug_log(":align before_wanted %ld", before_wanted); |
| 263 | } |
| 264 | |
| 265 | after_wanted = before_wanted - (time_t)(points_wanted * group * query_granularity) + query_granularity; |
| 266 | query_debug_log(":final after_wanted %ld", after_wanted); |
| 267 | |
| 268 | duration = before_wanted - after_wanted; |
| 269 | query_debug_log(":final duration %ld", duration + 1); |
| 270 | |
| 271 | query_debug_log_fin(); |
| 272 | |
| 273 | internal_error(points_wanted != duration / (query_granularity * group) + 1, |
| 274 | "QUERY: points_wanted %zu is not points %zu", |
| 275 | points_wanted, (size_t)(duration / (query_granularity * group) + 1)); |
| 276 | |
| 277 | internal_error(group < resampling_group, |
| 278 | "QUERY: group %zu is less than the desired group points %zu", |
| 279 | group, resampling_group); |
| 280 | |
| 281 | internal_error(group > resampling_group && group % resampling_group, |
| 282 | "QUERY: group %zu is not a multiple of the desired group points %zu", |
| 283 | group, resampling_group); |
| 284 | |
| 285 | // ------------------------------------------------------------------------- |
| 286 | // update QUERY_TARGET with our calculations |
| 287 | |
| 288 | qt->window.after = after_wanted; |
| 289 | qt->window.before = before_wanted; |
| 290 | qt->window.relative = relative_period_requested; |
| 291 | qt->window.points = points_wanted; |
| 292 | qt->window.group = group; |
| 293 | qt->window.time_group_method = group_method; |
| 294 | qt->window.time_group_options = qt->request.time_group_options; |
| 295 | qt->window.query_granularity = query_granularity; |
| 296 | qt->window.resampling_group = resampling_group; |
| 297 | qt->window.resampling_divisor = resampling_divisor; |
| 298 | qt->window.options = options; |
| 299 | qt->window.tier = tier; |
| 300 | qt->window.aligned = aligned; |
| 301 | |
| 302 | return true; |
| 303 | } |