| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "query-internal.h" |
| 4 | |
| 5 | static bool query_metric_is_valid_tier(QUERY_METRIC *qm, size_t tier) { |
| 6 | if(!qm->tiers[tier].smh || !qm->tiers[tier].db_first_time_s || !qm->tiers[tier].db_last_time_s || !qm->tiers[tier].db_update_every_s) |
| 7 | return false; |
| 8 | |
| 9 | return true; |
| 10 | } |
| 11 | |
| 12 | static size_t query_metric_first_working_tier(QUERY_METRIC *qm) { |
| 13 | for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) { |
| 14 | |
| 15 | // find the db time-range for this tier for all metrics |
| 16 | STORAGE_METRIC_HANDLE *smh = qm->tiers[tier].smh; |
| 17 | time_t first_time_s = qm->tiers[tier].db_first_time_s; |
| 18 | time_t last_time_s = qm->tiers[tier].db_last_time_s; |
| 19 | time_t update_every_s = qm->tiers[tier].db_update_every_s; |
| 20 | |
| 21 | if(!smh || !first_time_s || !last_time_s || !update_every_s) |
| 22 | continue; |
| 23 | |
| 24 | return tier; |
| 25 | } |
| 26 | |
| 27 | return 0; |
| 28 | } |
| 29 | |
| 30 | #define QUERY_PLAN_POINTS_WEIGHT_SCALE 1000000ULL |
| 31 | #define QUERY_PLAN_ACCEPTABLE_POINTS_NUMERATOR 1ULL |
| 32 | #define QUERY_PLAN_ACCEPTABLE_POINTS_DENOMINATOR 2ULL |
| 33 | |
| 34 | static bool query_metric_tier_overlaps_timeframe(QUERY_METRIC *qm, size_t tier, time_t after_wanted, time_t before_wanted) { |
| 35 | if(!query_metric_is_valid_tier(qm, tier)) |
| 36 | return false; |
| 37 | |
| 38 | return qm->tiers[tier].db_first_time_s <= before_wanted && |
| 39 | qm->tiers[tier].db_last_time_s >= after_wanted; |
| 40 | } |
| 41 | |
| 42 | static long query_plan_points_density_weight(time_t db_update_every_s, time_t after_wanted, time_t before_wanted) { |
| 43 | if(db_update_every_s <= 0 || before_wanted <= after_wanted) |
| 44 | return -LONG_MAX; |
| 45 | |
| 46 | uint64_t duration_s = (uint64_t)(before_wanted - after_wanted); |
| 47 | |
| 48 | if(duration_s > (uint64_t)LONG_MAX / QUERY_PLAN_POINTS_WEIGHT_SCALE) |
| 49 | return LONG_MAX; |
| 50 | |
| 51 | return (long)((duration_s * QUERY_PLAN_POINTS_WEIGHT_SCALE) / (uint64_t)db_update_every_s); |
| 52 | } |
| 53 | |
| 54 | static long query_plan_minimum_acceptable_points_weight(size_t points_wanted) { |
| 55 | if(!points_wanted) |
| 56 | return 0; |
| 57 | |
| 58 | if((uint64_t)points_wanted > (uint64_t)LONG_MAX / QUERY_PLAN_POINTS_WEIGHT_SCALE) |
| 59 | return LONG_MAX; |
| 60 | |
| 61 | uint64_t wanted_scaled = (uint64_t)points_wanted * QUERY_PLAN_POINTS_WEIGHT_SCALE; |
| 62 | |
| 63 | if(wanted_scaled > UINT64_MAX / QUERY_PLAN_ACCEPTABLE_POINTS_NUMERATOR) |
| 64 | return LONG_MAX; |
| 65 | |
| 66 | uint64_t acceptable_scaled = |
| 67 | (wanted_scaled * QUERY_PLAN_ACCEPTABLE_POINTS_NUMERATOR + QUERY_PLAN_ACCEPTABLE_POINTS_DENOMINATOR - 1) / |
| 68 | QUERY_PLAN_ACCEPTABLE_POINTS_DENOMINATOR; |
| 69 | |
| 70 | if(acceptable_scaled > (uint64_t)LONG_MAX) |
| 71 | return LONG_MAX; |
| 72 | |
| 73 | return (long)acceptable_scaled; |
| 74 | } |
| 75 | |
| 76 | static bool query_plan_points_density_is_better( |
| 77 | size_t tier, long weight, bool acceptable, |
| 78 | size_t best_tier, long best_weight, bool best_acceptable) { |
| 79 | if(acceptable) { |
| 80 | if(!best_acceptable) |
| 81 | return true; |
| 82 | |
| 83 | if(weight < best_weight) |
| 84 | return true; |
| 85 | |
| 86 | return weight == best_weight && tier > best_tier; |
| 87 | } |
| 88 | |
| 89 | if(best_acceptable) |
| 90 | return false; |
| 91 | |
| 92 | if(weight > best_weight) |
| 93 | return true; |
| 94 | |
| 95 | return weight == best_weight && tier < best_tier; |
| 96 | } |
| 97 | |
| 98 | static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t after_wanted, time_t before_wanted, size_t points_wanted) { |
| 99 | if(unlikely(nd_profile.storage_tiers < 2)) |
| 100 | return 0; |
| 101 | |
| 102 | if(unlikely(before_wanted <= after_wanted || points_wanted <= 0)) |
| 103 | return query_metric_first_working_tier(qm); |
| 104 | |
| 105 | if(points_wanted < QUERY_PLAN_MIN_POINTS) |
| 106 | // when selecting tiers, aim for a resolution of at least QUERY_PLAN_MIN_POINTS points |
| 107 | points_wanted = (before_wanted - after_wanted) > QUERY_PLAN_MIN_POINTS ? QUERY_PLAN_MIN_POINTS : before_wanted - after_wanted; |
| 108 | |
| 109 | long minimum_acceptable_weight = query_plan_minimum_acceptable_points_weight(points_wanted); |
| 110 | |
| 111 | size_t best_tier = 0; |
| 112 | long best_weight = -LONG_MAX; |
| 113 | bool best_acceptable = false; |
| 114 | bool found_candidate = false; |
| 115 | |
| 116 | for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) { |
| 117 | |
| 118 | time_t update_every_s = qm->tiers[tier].db_update_every_s; |
| 119 | |
| 120 | if(!query_metric_tier_overlaps_timeframe(qm, tier, after_wanted, before_wanted)) { |
| 121 | qm->tiers[tier].weight = -LONG_MAX; |
| 122 | continue; |
| 123 | } |
| 124 | |
| 125 | qm->tiers[tier].weight = query_plan_points_density_weight(update_every_s, after_wanted, before_wanted); |
| 126 | if(qm->tiers[tier].weight == -LONG_MAX) |
| 127 | continue; |
| 128 | |
| 129 | bool acceptable = qm->tiers[tier].weight >= minimum_acceptable_weight; |
| 130 | |
| 131 | if(!found_candidate || |
| 132 | query_plan_points_density_is_better( |
| 133 | tier, qm->tiers[tier].weight, acceptable, |
| 134 | best_tier, best_weight, best_acceptable)) { |
| 135 | best_tier = tier; |
| 136 | best_weight = qm->tiers[tier].weight; |
| 137 | best_acceptable = acceptable; |
| 138 | found_candidate = true; |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | return found_candidate ? best_tier : query_metric_first_working_tier(qm); |
| 143 | } |
| 144 | |
| 145 | time_t query_target_min_update_every_for_tier(QUERY_TARGET *qt, size_t tier) { |
| 146 | if(tier >= nd_profile.storage_tiers) |
| 147 | return nd_profile.update_every; |
| 148 | |
| 149 | // find the db minimum update every for this tier for all metrics |
| 150 | time_t common_update_every_s = 0; |
| 151 | for(size_t i = 0, used = qt->query.used; i < used ; i++) { |
| 152 | QUERY_METRIC *qm = query_metric(qt, i); |
| 153 | |
| 154 | time_t update_every_s = qm->tiers[tier].db_update_every_s; |
| 155 | if(!update_every_s) |
| 156 | continue; |
| 157 | |
| 158 | if(!common_update_every_s) |
| 159 | common_update_every_s = update_every_s; |
| 160 | else |
| 161 | common_update_every_s = MIN(update_every_s, common_update_every_s); |
| 162 | } |
| 163 | |
| 164 | return common_update_every_s ? common_update_every_s : nd_profile.update_every; |
| 165 | } |
| 166 | |
| 167 | static size_t query_planer_expand_duration_in_points(time_t this_update_every, time_t next_update_every) { |
| 168 | |
| 169 | time_t delta = this_update_every - next_update_every; |
| 170 | if(delta < 0) delta = -delta; |
| 171 | |
| 172 | size_t points; |
| 173 | if(delta < this_update_every * POINTS_TO_EXPAND_QUERY) |
| 174 | points = POINTS_TO_EXPAND_QUERY; |
| 175 | else |
| 176 | points = (delta + this_update_every - 1) / this_update_every; |
| 177 | |
| 178 | return points; |
| 179 | } |
| 180 | |
| 181 | static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) { |
| 182 | QUERY_METRIC *qm = ops->qm; |
| 183 | |
| 184 | for(size_t p = 0; p < qm->plan.used ; p++) { |
| 185 | size_t tier = qm->plan.array[p].tier; |
| 186 | time_t update_every = qm->tiers[tier].db_update_every_s; |
| 187 | |
| 188 | size_t points_to_add_to_after; |
| 189 | if(p > 0) { |
| 190 | // there is another plan before to this |
| 191 | |
| 192 | size_t tier0 = qm->plan.array[p - 1].tier; |
| 193 | time_t update_every0 = qm->tiers[tier0].db_update_every_s; |
| 194 | |
| 195 | points_to_add_to_after = query_planer_expand_duration_in_points(update_every, update_every0); |
| 196 | } |
| 197 | else |
| 198 | points_to_add_to_after = (tier == 0) ? 0 : POINTS_TO_EXPAND_QUERY; |
| 199 | |
| 200 | size_t points_to_add_to_before; |
| 201 | if(p + 1 < qm->plan.used) { |
| 202 | // there is another plan after to this |
| 203 | |
| 204 | size_t tier1 = qm->plan.array[p+1].tier; |
| 205 | time_t update_every1 = qm->tiers[tier1].db_update_every_s; |
| 206 | |
| 207 | points_to_add_to_before = query_planer_expand_duration_in_points(update_every, update_every1); |
| 208 | } |
| 209 | else |
| 210 | points_to_add_to_before = POINTS_TO_EXPAND_QUERY; |
| 211 | |
| 212 | time_t after = qm->plan.array[p].after - (time_t)(update_every * points_to_add_to_after); |
| 213 | time_t before = qm->plan.array[p].before + (time_t)(update_every * points_to_add_to_before); |
| 214 | |
| 215 | ops->plans[p].expanded_after = after; |
| 216 | ops->plans[p].expanded_before = before; |
| 217 | |
| 218 | ops->r->internal.qt->db.tiers[tier].queries++; |
| 219 | |
| 220 | struct query_metric_tier *tier_ptr = &qm->tiers[tier]; |
| 221 | STORAGE_ENGINE *eng = query_metric_storage_engine(ops->r->internal.qt, qm, tier); |
| 222 | storage_engine_query_init(eng->seb, tier_ptr->smh, &ops->plans[p].handle, |
| 223 | after, before, ops->r->internal.qt->request.priority); |
| 224 | |
| 225 | ops->plans[p].initialized = true; |
| 226 | ops->plans[p].finalized = false; |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | static void query_planer_finalize_plan(QUERY_ENGINE_OPS *ops, size_t plan_id) { |
| 231 | // QUERY_METRIC *qm = ops->qm; |
| 232 | |
| 233 | if(ops->plans[plan_id].initialized && !ops->plans[plan_id].finalized) { |
| 234 | storage_engine_query_finalize(&ops->plans[plan_id].handle); |
| 235 | ops->plans[plan_id].initialized = false; |
| 236 | ops->plans[plan_id].finalized = true; |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | void query_planer_finalize_remaining_plans(QUERY_ENGINE_OPS *ops) { |
| 241 | QUERY_METRIC *qm = ops->qm; |
| 242 | |
| 243 | for(size_t p = 0; p < qm->plan.used ; p++) |
| 244 | query_planer_finalize_plan(ops, p); |
| 245 | } |
| 246 | |
| 247 | static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, time_t overwrite_after __maybe_unused) { |
| 248 | QUERY_METRIC *qm = ops->qm; |
| 249 | |
| 250 | internal_fatal(plan_id >= qm->plan.used, "QUERY: invalid plan_id given"); |
| 251 | internal_fatal(!ops->plans[plan_id].initialized, "QUERY: plan has not been initialized"); |
| 252 | internal_fatal(ops->plans[plan_id].finalized, "QUERY: plan has been finalized"); |
| 253 | |
| 254 | internal_fatal(qm->plan.array[plan_id].after > qm->plan.array[plan_id].before, "QUERY: flipped after/before"); |
| 255 | |
| 256 | ops->tier = qm->plan.array[plan_id].tier; |
| 257 | ops->tier_ptr = &qm->tiers[ops->tier]; |
| 258 | ops->seqh = &ops->plans[plan_id].handle; |
| 259 | ops->current_plan = plan_id; |
| 260 | |
| 261 | if(plan_id + 1 < qm->plan.used && qm->plan.array[plan_id + 1].after < qm->plan.array[plan_id].before) |
| 262 | ops->current_plan_expire_time = qm->plan.array[plan_id + 1].after; |
| 263 | else |
| 264 | ops->current_plan_expire_time = qm->plan.array[plan_id].before; |
| 265 | |
| 266 | ops->plan_expanded_after = ops->plans[plan_id].expanded_after; |
| 267 | ops->plan_expanded_before = ops->plans[plan_id].expanded_before; |
| 268 | } |
| 269 | |
| 270 | bool query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) { |
| 271 | QUERY_METRIC *qm = ops->qm; |
| 272 | |
| 273 | size_t old_plan = ops->current_plan; |
| 274 | |
| 275 | time_t next_plan_before_time; |
| 276 | do { |
| 277 | ops->current_plan++; |
| 278 | |
| 279 | if (ops->current_plan >= qm->plan.used) { |
| 280 | ops->current_plan = old_plan; |
| 281 | ops->current_plan_expire_time = ops->r->internal.qt->window.before; |
| 282 | // let the query run with current plan |
| 283 | // we will not switch it |
| 284 | return false; |
| 285 | } |
| 286 | |
| 287 | next_plan_before_time = qm->plan.array[ops->current_plan].before; |
| 288 | } while(now >= next_plan_before_time || last_point_end_time >= next_plan_before_time); |
| 289 | |
| 290 | if(!query_metric_is_valid_tier(qm, qm->plan.array[ops->current_plan].tier)) { |
| 291 | ops->current_plan = old_plan; |
| 292 | ops->current_plan_expire_time = ops->r->internal.qt->window.before; |
| 293 | return false; |
| 294 | } |
| 295 | |
| 296 | query_planer_finalize_plan(ops, old_plan); |
| 297 | query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time)); |
| 298 | return true; |
| 299 | } |
| 300 | |
| 301 | static int compare_query_plan_entries_on_start_time(const void *a, const void *b) { |
| 302 | QUERY_PLAN_ENTRY *p1 = (QUERY_PLAN_ENTRY *)a; |
| 303 | QUERY_PLAN_ENTRY *p2 = (QUERY_PLAN_ENTRY *)b; |
| 304 | return (p1->after < p2->after)?-1:1; |
| 305 | } |
| 306 | |
| 307 | static bool query_plan_build_entries(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, size_t points_wanted) { |
| 308 | QUERY_METRIC *qm = ops->qm; |
| 309 | |
| 310 | // put our selected tier as the first plan |
| 311 | size_t selected_tier; |
| 312 | bool switch_tiers = true; |
| 313 | |
| 314 | if((ops->r->internal.qt->window.options & RRDR_OPTION_SELECTED_TIER) |
| 315 | && ops->r->internal.qt->window.tier < nd_profile.storage_tiers && query_metric_is_valid_tier(qm, ops->r->internal.qt->window.tier)) { |
| 316 | selected_tier = ops->r->internal.qt->window.tier; |
| 317 | switch_tiers = false; |
| 318 | } |
| 319 | else { |
| 320 | selected_tier = query_metric_best_tier_for_timeframe(qm, after_wanted, before_wanted, points_wanted); |
| 321 | |
| 322 | if(!query_metric_is_valid_tier(qm, selected_tier)) |
| 323 | return false; |
| 324 | } |
| 325 | |
| 326 | if(qm->tiers[selected_tier].db_first_time_s > before_wanted || |
| 327 | qm->tiers[selected_tier].db_last_time_s < after_wanted) { |
| 328 | // we don't have any data to satisfy this query |
| 329 | return false; |
| 330 | } |
| 331 | |
| 332 | qm->plan.used = 1; |
| 333 | qm->plan.array[0].tier = selected_tier; |
| 334 | qm->plan.array[0].after = (qm->tiers[selected_tier].db_first_time_s < after_wanted) ? after_wanted : qm->tiers[selected_tier].db_first_time_s; |
| 335 | qm->plan.array[0].before = (qm->tiers[selected_tier].db_last_time_s > before_wanted) ? before_wanted : qm->tiers[selected_tier].db_last_time_s; |
| 336 | |
| 337 | if(switch_tiers) { |
| 338 | // the selected tier |
| 339 | time_t selected_tier_first_time_s = qm->plan.array[0].after; |
| 340 | time_t selected_tier_last_time_s = qm->plan.array[0].before; |
| 341 | |
| 342 | // check if our selected tier can start the query |
| 343 | if (selected_tier_first_time_s > after_wanted) { |
| 344 | // we need some help from other tiers |
| 345 | for (size_t tr = (int)selected_tier + 1; tr < nd_profile.storage_tiers && qm->plan.used < QUERY_PLANS_MAX ; tr++) { |
| 346 | if(!query_metric_is_valid_tier(qm, tr)) |
| 347 | continue; |
| 348 | |
| 349 | // find the first time of this tier |
| 350 | time_t tier_first_time_s = qm->tiers[tr].db_first_time_s; |
| 351 | time_t tier_last_time_s = qm->tiers[tr].db_last_time_s; |
| 352 | |
| 353 | // can it help? |
| 354 | if (tier_first_time_s < selected_tier_first_time_s && tier_first_time_s <= before_wanted && tier_last_time_s >= after_wanted) { |
| 355 | // it can help us add detail at the beginning of the query |
| 356 | QUERY_PLAN_ENTRY t = { |
| 357 | .tier = tr, |
| 358 | .after = (tier_first_time_s < after_wanted) ? after_wanted : tier_first_time_s, |
| 359 | .before = selected_tier_first_time_s, |
| 360 | }; |
| 361 | ops->plans[qm->plan.used].initialized = false; |
| 362 | ops->plans[qm->plan.used].finalized = false; |
| 363 | qm->plan.array[qm->plan.used++] = t; |
| 364 | |
| 365 | internal_fatal(!t.after || !t.before, "QUERY: invalid plan selected"); |
| 366 | |
| 367 | // prepare for the tier |
| 368 | selected_tier_first_time_s = t.after; |
| 369 | |
| 370 | if (t.after <= after_wanted) |
| 371 | break; |
| 372 | } |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | // check if our selected tier can finish the query |
| 377 | if (selected_tier_last_time_s < before_wanted) { |
| 378 | // we need some help from other tiers |
| 379 | for (int tr = (int)selected_tier - 1; tr >= 0 && qm->plan.used < QUERY_PLANS_MAX ; tr--) { |
| 380 | if(!query_metric_is_valid_tier(qm, tr)) |
| 381 | continue; |
| 382 | |
| 383 | // find the last time of this tier |
| 384 | time_t tier_first_time_s = qm->tiers[tr].db_first_time_s; |
| 385 | time_t tier_last_time_s = qm->tiers[tr].db_last_time_s; |
| 386 | |
| 387 | //buffer_sprintf(wb, ": EVAL BEFORE tier %d, %ld", tier, last_time_s); |
| 388 | |
| 389 | // can it help? |
| 390 | if (tier_last_time_s > selected_tier_last_time_s && tier_first_time_s <= before_wanted && tier_last_time_s >= after_wanted) { |
| 391 | // it can help us add detail at the end of the query |
| 392 | QUERY_PLAN_ENTRY t = { |
| 393 | .tier = tr, |
| 394 | .after = selected_tier_last_time_s, |
| 395 | .before = (tier_last_time_s > before_wanted) ? before_wanted : tier_last_time_s, |
| 396 | }; |
| 397 | ops->plans[qm->plan.used].initialized = false; |
| 398 | ops->plans[qm->plan.used].finalized = false; |
| 399 | qm->plan.array[qm->plan.used++] = t; |
| 400 | |
| 401 | // prepare for the tier |
| 402 | selected_tier_last_time_s = t.before; |
| 403 | |
| 404 | internal_fatal(!t.after || !t.before, "QUERY: invalid plan selected"); |
| 405 | |
| 406 | if (t.before >= before_wanted) |
| 407 | break; |
| 408 | } |
| 409 | } |
| 410 | } |
| 411 | } |
| 412 | |
| 413 | // sort the query plan |
| 414 | if(qm->plan.used > 1) |
| 415 | qsort(&qm->plan.array, qm->plan.used, sizeof(QUERY_PLAN_ENTRY), compare_query_plan_entries_on_start_time); |
| 416 | |
| 417 | if(!query_metric_is_valid_tier(qm, qm->plan.array[0].tier)) |
| 418 | return false; |
| 419 | |
| 420 | #ifdef NETDATA_INTERNAL_CHECKS |
| 421 | for(size_t p = 0; p < qm->plan.used ;p++) { |
| 422 | internal_fatal(qm->plan.array[p].after > qm->plan.array[p].before, "QUERY: flipped after/before"); |
| 423 | internal_fatal(qm->plan.array[p].after < after_wanted, "QUERY: too small plan first time"); |
| 424 | internal_fatal(qm->plan.array[p].before > before_wanted, "QUERY: too big plan last time"); |
| 425 | } |
| 426 | #endif |
| 427 | |
| 428 | return true; |
| 429 | } |
| 430 | |
| 431 | static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, size_t points_wanted) { |
| 432 | if(!query_plan_build_entries(ops, after_wanted, before_wanted, points_wanted)) |
| 433 | return false; |
| 434 | |
| 435 | query_planer_initialize_plans(ops); |
| 436 | query_planer_activate_plan(ops, 0, 0); |
| 437 | |
| 438 | return true; |
| 439 | } |
| 440 | |
| 441 | |
| 442 | static __thread QUERY_ENGINE_OPS *released_ops = NULL; |
| 443 | |
| 444 | void rrd2rrdr_query_ops_freeall(RRDR *r __maybe_unused) { |
| 445 | while(released_ops) { |
| 446 | QUERY_ENGINE_OPS *ops = released_ops; |
| 447 | released_ops = ops->next; |
| 448 | |
| 449 | onewayalloc_freez(r->internal.owa, ops); |
| 450 | } |
| 451 | } |
| 452 | |
| 453 | void rrd2rrdr_query_ops_release(QUERY_ENGINE_OPS *ops) { |
| 454 | if(!ops) return; |
| 455 | |
| 456 | ops->next = released_ops; |
| 457 | released_ops = ops; |
| 458 | } |
| 459 | |
| 460 | static QUERY_ENGINE_OPS *rrd2rrdr_query_ops_get(RRDR *r) { |
| 461 | QUERY_ENGINE_OPS *ops; |
| 462 | if(released_ops) { |
| 463 | ops = released_ops; |
| 464 | released_ops = ops->next; |
| 465 | } |
| 466 | else { |
| 467 | ops = onewayalloc_mallocz(r->internal.owa, sizeof(QUERY_ENGINE_OPS)); |
| 468 | } |
| 469 | |
| 470 | memset(ops, 0, sizeof(*ops)); |
| 471 | return ops; |
| 472 | } |
| 473 | |
| 474 | QUERY_ENGINE_OPS *rrd2rrdr_query_ops_prep(RRDR *r, size_t query_metric_id) { |
| 475 | QUERY_TARGET *qt = r->internal.qt; |
| 476 | |
| 477 | QUERY_ENGINE_OPS *ops = rrd2rrdr_query_ops_get(r); |
| 478 | *ops = (QUERY_ENGINE_OPS) { |
| 479 | .r = r, |
| 480 | .qm = query_metric(qt, query_metric_id), |
| 481 | .tier_query_fetch = r->time_grouping.tier_query_fetch, |
| 482 | .view_update_every = r->view.update_every, |
| 483 | .query_granularity = (time_t)(r->view.update_every / r->view.group), |
| 484 | .group_value_flags = RRDR_VALUE_NOTHING, |
| 485 | }; |
| 486 | |
| 487 | if(!query_plan(ops, qt->window.after, qt->window.before, qt->window.points)) { |
| 488 | rrd2rrdr_query_ops_release(ops); |
| 489 | return NULL; |
| 490 | } |
| 491 | |
| 492 | return ops; |
| 493 | } |
| 494 | |
| 495 | static void query_plan_unittest_set_tier( |
| 496 | QUERY_METRIC *qm, size_t tier, time_t first_time_s, time_t last_time_s, time_t update_every_s) { |
| 497 | static char smh_stub; |
| 498 | |
| 499 | qm->tiers[tier].smh = (STORAGE_METRIC_HANDLE *)&smh_stub; |
| 500 | qm->tiers[tier].db_first_time_s = first_time_s; |
| 501 | qm->tiers[tier].db_last_time_s = last_time_s; |
| 502 | qm->tiers[tier].db_update_every_s = update_every_s; |
| 503 | } |
| 504 | |
| 505 | static int query_plan_unittest_expect_best_tier( |
| 506 | const char *name, QUERY_METRIC *qm, time_t after, time_t before, size_t points, size_t expected) { |
| 507 | size_t got = query_metric_best_tier_for_timeframe(qm, after, before, points); |
| 508 | if(got == expected) { |
| 509 | fprintf(stderr, "OK query plan tier selection: %s\n", name); |
| 510 | return 0; |
| 511 | } |
| 512 | |
| 513 | fprintf(stderr, |
| 514 | "FAILED query plan tier selection: %s, expected tier %zu, got tier %zu\n", |
| 515 | name, expected, got); |
| 516 | |
| 517 | for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) |
| 518 | fprintf(stderr, |
| 519 | " tier %zu: first %ld, last %ld, update_every %ld, weight %ld\n", |
| 520 | tier, |
| 521 | qm->tiers[tier].db_first_time_s, |
| 522 | qm->tiers[tier].db_last_time_s, |
| 523 | qm->tiers[tier].db_update_every_s, |
| 524 | qm->tiers[tier].weight); |
| 525 | |
| 526 | return 1; |
| 527 | } |
| 528 | |
| 529 | static bool query_plan_unittest_build_entries( |
| 530 | QUERY_METRIC *qm, RRDR_OPTIONS options, size_t selected_tier, |
| 531 | time_t after, time_t before, size_t points) { |
| 532 | RRDR r = {0}; |
| 533 | QUERY_TARGET qt = {0}; |
| 534 | QUERY_ENGINE_OPS ops = { |
| 535 | .r = &r, |
| 536 | .qm = qm, |
| 537 | }; |
| 538 | |
| 539 | r.internal.qt = &qt; |
| 540 | qt.window.options = options; |
| 541 | qt.window.tier = selected_tier; |
| 542 | |
| 543 | return query_plan_build_entries(&ops, after, before, points); |
| 544 | } |
| 545 | |
| 546 | static int query_plan_unittest_expect_plan( |
| 547 | const char *name, QUERY_METRIC *qm, RRDR_OPTIONS options, size_t selected_tier, |
| 548 | time_t after, time_t before, size_t points, |
| 549 | const QUERY_PLAN_ENTRY *expected, size_t expected_used) { |
| 550 | if(!query_plan_unittest_build_entries(qm, options, selected_tier, after, before, points)) { |
| 551 | fprintf(stderr, "FAILED query plan entries: %s, planner returned false\n", name); |
| 552 | return 1; |
| 553 | } |
| 554 | |
| 555 | if(qm->plan.used != expected_used) { |
| 556 | fprintf(stderr, |
| 557 | "FAILED query plan entries: %s, expected %zu entries, got %zu\n", |
| 558 | name, expected_used, qm->plan.used); |
| 559 | return 1; |
| 560 | } |
| 561 | |
| 562 | for(size_t i = 0; i < expected_used; i++) { |
| 563 | if(qm->plan.array[i].tier == expected[i].tier && |
| 564 | qm->plan.array[i].after == expected[i].after && |
| 565 | qm->plan.array[i].before == expected[i].before) |
| 566 | continue; |
| 567 | |
| 568 | fprintf(stderr, |
| 569 | "FAILED query plan entries: %s, entry %zu expected tier %zu after %ld before %ld, got tier %zu after %ld before %ld\n", |
| 570 | name, i, |
| 571 | expected[i].tier, expected[i].after, expected[i].before, |
| 572 | qm->plan.array[i].tier, qm->plan.array[i].after, qm->plan.array[i].before); |
| 573 | return 1; |
| 574 | } |
| 575 | |
| 576 | fprintf(stderr, "OK query plan entries: %s\n", name); |
| 577 | return 0; |
| 578 | } |
| 579 | |
| 580 | static int query_plan_unittest_expect_no_plan( |
| 581 | const char *name, QUERY_METRIC *qm, RRDR_OPTIONS options, size_t selected_tier, |
| 582 | time_t after, time_t before, size_t points) { |
| 583 | if(!query_plan_unittest_build_entries(qm, options, selected_tier, after, before, points)) { |
| 584 | fprintf(stderr, "OK query plan entries: %s\n", name); |
| 585 | return 0; |
| 586 | } |
| 587 | |
| 588 | fprintf(stderr, "FAILED query plan entries: %s, expected no plan, got %zu entries\n", name, qm->plan.used); |
| 589 | return 1; |
| 590 | } |
| 591 | |
| 592 | static int query_plan_unittest_expect_update_every(QUERY_TARGET *qt, size_t tier, time_t expected) { |
| 593 | time_t got = query_target_min_update_every_for_tier(qt, tier); |
| 594 | if(got == expected) { |
| 595 | fprintf(stderr, "OK query plan selected-tier natural update_every\n"); |
| 596 | return 0; |
| 597 | } |
| 598 | |
| 599 | fprintf(stderr, |
| 600 | "FAILED query plan selected-tier natural update_every: expected %ld, got %ld\n", |
| 601 | expected, got); |
| 602 | |
| 603 | return 1; |
| 604 | } |
| 605 | |
| 606 | int query_plan_unittest(void) { |
| 607 | size_t old_storage_tiers = nd_profile.storage_tiers; |
| 608 | time_t old_update_every = nd_profile.update_every; |
| 609 | int errors = 0; |
| 610 | |
| 611 | nd_profile.storage_tiers = 3; |
| 612 | nd_profile.update_every = 1; |
| 613 | |
| 614 | { |
| 615 | QUERY_METRIC qm = {0}; |
| 616 | query_plan_unittest_set_tier(&qm, 0, 1, 200, 10); |
| 617 | query_plan_unittest_set_tier(&qm, 1, 1, 200, 600); |
| 618 | query_plan_unittest_set_tier(&qm, 2, 1, 100, 36000); |
| 619 | |
| 620 | errors += query_plan_unittest_expect_best_tier( |
| 621 | "sub-resolution window ignores non-overlapping coarser tier", &qm, 103, 108, 5, 0); |
| 622 | } |
| 623 | |
| 624 | { |
| 625 | QUERY_METRIC qm = {0}; |
| 626 | query_plan_unittest_set_tier(&qm, 0, 1, 200, 10); |
| 627 | query_plan_unittest_set_tier(&qm, 1, 1, 200, 600); |
| 628 | query_plan_unittest_set_tier(&qm, 2, 1, 200, 36000); |
| 629 | |
| 630 | errors += query_plan_unittest_expect_best_tier( |
| 631 | "sub-resolution window chooses densest overlapping tier", &qm, 103, 108, 5, 0); |
| 632 | } |
| 633 | |
| 634 | { |
| 635 | QUERY_METRIC qm = {0}; |
| 636 | query_plan_unittest_set_tier(&qm, 0, 1, 400000, 1); |
| 637 | query_plan_unittest_set_tier(&qm, 1, 1, 400000, 600); |
| 638 | query_plan_unittest_set_tier(&qm, 2, 1, 400000, 36000); |
| 639 | |
| 640 | errors += query_plan_unittest_expect_best_tier( |
| 641 | "50 percent tolerance chooses sparsest acceptable tier", &qm, 1000, 301000, 500, 1); |
| 642 | } |
| 643 | |
| 644 | { |
| 645 | QUERY_METRIC qm = {0}; |
| 646 | query_plan_unittest_set_tier(&qm, 0, 1, 1000, 1); |
| 647 | query_plan_unittest_set_tier(&qm, 1, 1, 1000, 10); |
| 648 | query_plan_unittest_set_tier(&qm, 2, 1, 1000, 11); |
| 649 | |
| 650 | errors += query_plan_unittest_expect_best_tier( |
| 651 | "50 percent tolerance includes exact threshold", &qm, 100, 400, 60, 1); |
| 652 | } |
| 653 | |
| 654 | { |
| 655 | QUERY_METRIC qm = {0}; |
| 656 | query_plan_unittest_set_tier(&qm, 0, 1, 1000, 10); |
| 657 | query_plan_unittest_set_tier(&qm, 1, 1, 1000, 600); |
| 658 | query_plan_unittest_set_tier(&qm, 2, 1, 1000, 36000); |
| 659 | |
| 660 | errors += query_plan_unittest_expect_best_tier( |
| 661 | "under-resolution request chooses densest tier", &qm, 100, 700, 600, 0); |
| 662 | } |
| 663 | |
| 664 | { |
| 665 | QUERY_METRIC qm = {0}; |
| 666 | query_plan_unittest_set_tier(&qm, 0, 1, 50, 10); |
| 667 | query_plan_unittest_set_tier(&qm, 1, 100, 200, 600); |
| 668 | query_plan_unittest_set_tier(&qm, 2, 1, 50, 36000); |
| 669 | |
| 670 | errors += query_plan_unittest_expect_best_tier( |
| 671 | "zero-overlap tiers are not candidates", &qm, 103, 108, 5, 1); |
| 672 | } |
| 673 | |
| 674 | { |
| 675 | QUERY_METRIC qm = {0}; |
| 676 | query_plan_unittest_set_tier(&qm, 1, 1, 200, 600); |
| 677 | query_plan_unittest_set_tier(&qm, 2, 1, 200, 36000); |
| 678 | |
| 679 | errors += query_plan_unittest_expect_best_tier( |
| 680 | "invalid duration returns first working tier", &qm, 108, 108, 5, 1); |
| 681 | } |
| 682 | |
| 683 | { |
| 684 | QUERY_METRIC qm = {0}; |
| 685 | query_plan_unittest_set_tier(&qm, 0, 1, 300, 10); |
| 686 | query_plan_unittest_set_tier(&qm, 1, 1, 300, 600); |
| 687 | |
| 688 | QUERY_PLAN_ENTRY expected[] = { |
| 689 | { .tier = 0, .after = 100, .before = 200 }, |
| 690 | }; |
| 691 | |
| 692 | errors += query_plan_unittest_expect_plan( |
| 693 | "selected tier covers full window", &qm, 0, 0, 100, 200, 10, expected, _countof(expected)); |
| 694 | } |
| 695 | |
| 696 | { |
| 697 | QUERY_METRIC qm = {0}; |
| 698 | query_plan_unittest_set_tier(&qm, 0, 100, 180, 10); |
| 699 | query_plan_unittest_set_tier(&qm, 1, 50, 150, 30); |
| 700 | |
| 701 | QUERY_PLAN_ENTRY expected[] = { |
| 702 | { .tier = 1, .after = 50, .before = 100 }, |
| 703 | { .tier = 0, .after = 100, .before = 180 }, |
| 704 | }; |
| 705 | |
| 706 | errors += query_plan_unittest_expect_plan( |
| 707 | "coarser tier fills head gap", &qm, 0, 0, 50, 180, 10, expected, _countof(expected)); |
| 708 | } |
| 709 | |
| 710 | { |
| 711 | QUERY_METRIC qm = {0}; |
| 712 | query_plan_unittest_set_tier(&qm, 0, 180, 260, 10); |
| 713 | query_plan_unittest_set_tier(&qm, 1, 100, 200, 30); |
| 714 | |
| 715 | QUERY_PLAN_ENTRY expected[] = { |
| 716 | { .tier = 1, .after = 100, .before = 200 }, |
| 717 | { .tier = 0, .after = 200, .before = 250 }, |
| 718 | }; |
| 719 | |
| 720 | errors += query_plan_unittest_expect_plan( |
| 721 | "finer tier fills tail gap", &qm, 0, 0, 100, 250, 10, expected, _countof(expected)); |
| 722 | } |
| 723 | |
| 724 | { |
| 725 | QUERY_METRIC qm = {0}; |
| 726 | query_plan_unittest_set_tier(&qm, 0, 180, 260, 10); |
| 727 | query_plan_unittest_set_tier(&qm, 1, 100, 200, 30); |
| 728 | query_plan_unittest_set_tier(&qm, 2, 50, 150, 60); |
| 729 | |
| 730 | QUERY_PLAN_ENTRY expected[] = { |
| 731 | { .tier = 2, .after = 50, .before = 100 }, |
| 732 | { .tier = 1, .after = 100, .before = 200 }, |
| 733 | { .tier = 0, .after = 200, .before = 250 }, |
| 734 | }; |
| 735 | |
| 736 | errors += query_plan_unittest_expect_plan( |
| 737 | "planner fills both head and tail gaps", &qm, 0, 0, 50, 250, 10, expected, _countof(expected)); |
| 738 | } |
| 739 | |
| 740 | { |
| 741 | QUERY_METRIC qm = {0}; |
| 742 | query_plan_unittest_set_tier(&qm, 0, 180, 260, 10); |
| 743 | query_plan_unittest_set_tier(&qm, 1, 100, 200, 30); |
| 744 | query_plan_unittest_set_tier(&qm, 2, 50, 150, 60); |
| 745 | |
| 746 | QUERY_PLAN_ENTRY expected[] = { |
| 747 | { .tier = 1, .after = 100, .before = 200 }, |
| 748 | }; |
| 749 | |
| 750 | errors += query_plan_unittest_expect_plan( |
| 751 | "explicit selected tier disables gap filling", &qm, RRDR_OPTION_SELECTED_TIER, 1, |
| 752 | 50, 250, 10, expected, _countof(expected)); |
| 753 | } |
| 754 | |
| 755 | { |
| 756 | QUERY_METRIC qm = {0}; |
| 757 | query_plan_unittest_set_tier(&qm, 0, 1, 50, 10); |
| 758 | query_plan_unittest_set_tier(&qm, 1, 60, 90, 30); |
| 759 | query_plan_unittest_set_tier(&qm, 2, 100, 150, 60); |
| 760 | |
| 761 | errors += query_plan_unittest_expect_no_plan( |
| 762 | "no overlapping tier fails planning", &qm, 0, 0, 200, 250, 10); |
| 763 | } |
| 764 | |
| 765 | { |
| 766 | QUERY_METRIC metrics[2] = {0}; |
| 767 | QUERY_TARGET qt = {0}; |
| 768 | |
| 769 | metrics[0].tiers[1].db_update_every_s = 600; |
| 770 | metrics[1].tiers[1].db_update_every_s = 300; |
| 771 | qt.query.array = metrics; |
| 772 | qt.query.used = 2; |
| 773 | |
| 774 | errors += query_plan_unittest_expect_update_every(&qt, 1, 300); |
| 775 | } |
| 776 | |
| 777 | nd_profile.storage_tiers = old_storage_tiers; |
| 778 | nd_profile.update_every = old_update_every; |
| 779 | |
| 780 | return errors; |
| 781 | } |