| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "rrdengine.h" |
| 4 | |
| 5 | MRG *main_mrg = NULL; |
| 6 | PGC *main_cache = NULL; |
| 7 | PGC *open_cache = NULL; |
| 8 | PGC *extent_cache = NULL; |
| 9 | struct rrdeng_cache_efficiency_stats rrdeng_cache_efficiency_stats = {}; |
| 10 | |
| 11 | static void main_cache_free_clean_page_callback(PGC *cache __maybe_unused, PGC_ENTRY entry __maybe_unused) |
| 12 | { |
| 13 | // Release storage associated with the page |
| 14 | pgd_free(entry.data); |
| 15 | } |
| 16 | |
| 17 | static void main_cache_flush_dirty_page_init_callback(PGC *cache __maybe_unused, Word_t section) { |
| 18 | struct rrdengine_instance *ctx = (struct rrdengine_instance *) section; |
| 19 | |
| 20 | // mark ctx as having flushing in progress |
| 21 | __atomic_add_fetch(&ctx->atomic.extents_currently_being_flushed, 1, __ATOMIC_RELAXED); |
| 22 | } |
| 23 | |
| 24 | static void main_cache_flush_dirty_page_callback(PGC *cache __maybe_unused, PGC_ENTRY *entries_array __maybe_unused, PGC_PAGE **pages_array __maybe_unused, size_t entries __maybe_unused) |
| 25 | { |
| 26 | if(!entries) |
| 27 | return; |
| 28 | |
| 29 | struct rrdengine_instance *ctx = (struct rrdengine_instance *) entries_array[0].section; |
| 30 | |
| 31 | struct page_descr_with_data *base = NULL; |
| 32 | |
| 33 | for (size_t Index = 0 ; Index < entries; Index++) { |
| 34 | time_t start_time_s = entries_array[Index].start_time_s; |
| 35 | time_t end_time_s = entries_array[Index].end_time_s; |
| 36 | struct page_descr_with_data *descr = page_descriptor_get(); |
| 37 | |
| 38 | descr->uuid_id = mrg_metric_uuidmap_id_dup(main_mrg, (METRIC *) entries_array[Index].metric_id); |
| 39 | descr->metric_id = entries_array[Index].metric_id; |
| 40 | descr->start_time_ut = start_time_s * USEC_PER_SEC; |
| 41 | descr->end_time_ut = end_time_s * USEC_PER_SEC; |
| 42 | descr->update_every_s = entries_array[Index].update_every_s; |
| 43 | |
| 44 | descr->pgd = pgc_page_data(pages_array[Index]); |
| 45 | descr->type = pgd_type(descr->pgd); |
| 46 | descr->page_length = pgd_disk_footprint(descr->pgd); |
| 47 | |
| 48 | DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(base, descr, link.prev, link.next); |
| 49 | } |
| 50 | |
| 51 | struct completion completion; |
| 52 | completion_init(&completion); |
| 53 | rrdeng_enq_cmd(ctx, RRDENG_OPCODE_EXTENT_WRITE, base, &completion, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL); |
| 54 | completion_wait_for(&completion); |
| 55 | completion_destroy(&completion); |
| 56 | } |
| 57 | |
| 58 | static void open_cache_free_clean_page_callback(PGC *cache __maybe_unused, PGC_ENTRY entry __maybe_unused) |
| 59 | { |
| 60 | struct rrdengine_datafile *datafile = entry.data; |
| 61 | datafile_release(datafile, DATAFILE_ACQUIRE_OPEN_CACHE); |
| 62 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_OPEN); |
| 63 | } |
| 64 | |
| 65 | static void open_cache_flush_dirty_page_callback(PGC *cache __maybe_unused, PGC_ENTRY *entries_array __maybe_unused, PGC_PAGE **pages_array __maybe_unused, size_t entries __maybe_unused) |
| 66 | { |
| 67 | ; |
| 68 | } |
| 69 | |
| 70 | static void extent_cache_free_clean_page_callback(PGC *cache __maybe_unused, PGC_ENTRY entry __maybe_unused) |
| 71 | { |
| 72 | dbengine_extent_free(entry.data, entry.size); |
| 73 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_EXTENT); |
| 74 | } |
| 75 | |
| 76 | static void extent_cache_flush_dirty_page_callback(PGC *cache __maybe_unused, PGC_ENTRY *entries_array __maybe_unused, PGC_PAGE **pages_array __maybe_unused, size_t entries __maybe_unused) |
| 77 | { |
| 78 | ; |
| 79 | } |
| 80 | |
| 81 | ALWAYS_INLINE_HOT TIME_RANGE_COMPARE is_page_in_time_range(time_t page_first_time_s, time_t page_last_time_s, time_t wanted_start_time_s, time_t wanted_end_time_s) { |
| 82 | // page_first_time_s <= wanted_end_time_s && page_last_time_s >= wanted_start_time_s |
| 83 | |
| 84 | if(page_last_time_s < wanted_start_time_s) |
| 85 | return PAGE_IS_IN_THE_PAST; |
| 86 | |
| 87 | if(page_first_time_s > wanted_end_time_s) |
| 88 | return PAGE_IS_IN_THE_FUTURE; |
| 89 | |
| 90 | return PAGE_IS_IN_RANGE; |
| 91 | } |
| 92 | |
| 93 | static ALWAYS_INLINE_HOT struct page_details *pdc_find_page_for_time( |
| 94 | Pcvoid_t PArray, |
| 95 | time_t wanted_time_s, |
| 96 | size_t *gaps, |
| 97 | PDC_PAGE_STATUS mode, |
| 98 | PDC_PAGE_STATUS skip_list |
| 99 | ) { |
| 100 | Word_t PIndexF = wanted_time_s, PIndexL = wanted_time_s; |
| 101 | Pvoid_t *PValueF, *PValueL; |
| 102 | struct page_details *pdF = NULL, *pdL = NULL; |
| 103 | bool firstF = true, firstL = true; |
| 104 | |
| 105 | PDC_PAGE_STATUS ignore_list = PDC_PAGE_QUERY_GLOBAL_SKIP_LIST | skip_list; |
| 106 | |
| 107 | while ((PValueF = PDCJudyLFirstThenNext(PArray, &PIndexF, &firstF))) { |
| 108 | pdF = *PValueF; |
| 109 | |
| 110 | PDC_PAGE_STATUS status = __atomic_load_n(&pdF->status, __ATOMIC_ACQUIRE); |
| 111 | if (!(status & (ignore_list | mode))) |
| 112 | break; |
| 113 | |
| 114 | pdF = NULL; |
| 115 | } |
| 116 | |
| 117 | while ((PValueL = PDCJudyLLastThenPrev(PArray, &PIndexL, &firstL))) { |
| 118 | pdL = *PValueL; |
| 119 | |
| 120 | PDC_PAGE_STATUS status = __atomic_load_n(&pdL->status, __ATOMIC_ACQUIRE); |
| 121 | if(status & mode) { |
| 122 | // don't go all the way back to the beginning |
| 123 | // stop at the last processed |
| 124 | pdL = NULL; |
| 125 | break; |
| 126 | } |
| 127 | |
| 128 | if (!(status & ignore_list)) |
| 129 | break; |
| 130 | |
| 131 | pdL = NULL; |
| 132 | } |
| 133 | |
| 134 | TIME_RANGE_COMPARE rcF = (pdF) ? is_page_in_time_range(pdF->first_time_s, pdF->last_time_s, wanted_time_s, wanted_time_s) : PAGE_IS_IN_THE_FUTURE; |
| 135 | TIME_RANGE_COMPARE rcL = (pdL) ? is_page_in_time_range(pdL->first_time_s, pdL->last_time_s, wanted_time_s, wanted_time_s) : PAGE_IS_IN_THE_PAST; |
| 136 | |
| 137 | if (!pdF || pdF == pdL) { |
| 138 | // F is missing, or they are the same |
| 139 | // return L |
| 140 | (*gaps) += (rcL == PAGE_IS_IN_RANGE) ? 0 : 1; |
| 141 | return pdL; |
| 142 | } |
| 143 | |
| 144 | if (!pdL) { |
| 145 | // L is missing |
| 146 | // return F |
| 147 | (*gaps) += (rcF == PAGE_IS_IN_RANGE) ? 0 : 1; |
| 148 | return pdF; |
| 149 | } |
| 150 | |
| 151 | if (rcF == rcL) { |
| 152 | // both are on the same side, |
| 153 | // but they are different pages |
| 154 | |
| 155 | switch (rcF) { |
| 156 | case PAGE_IS_IN_RANGE: |
| 157 | // pick the higher resolution |
| 158 | if (pdF->update_every_s && pdF->update_every_s < pdL->update_every_s) |
| 159 | return pdF; |
| 160 | |
| 161 | if (pdL->update_every_s && pdL->update_every_s < pdF->update_every_s) |
| 162 | return pdL; |
| 163 | |
| 164 | // same resolution - pick the one that starts earlier |
| 165 | if (pdL->first_time_s < pdF->first_time_s) |
| 166 | return pdL; |
| 167 | |
| 168 | return pdF; |
| 169 | break; |
| 170 | |
| 171 | case PAGE_IS_IN_THE_FUTURE: |
| 172 | (*gaps)++; |
| 173 | |
| 174 | // pick the one that starts earlier |
| 175 | if (pdL->first_time_s < pdF->first_time_s) |
| 176 | return pdL; |
| 177 | |
| 178 | return pdF; |
| 179 | break; |
| 180 | |
| 181 | default: |
| 182 | case PAGE_IS_IN_THE_PAST: |
| 183 | (*gaps)++; |
| 184 | return NULL; |
| 185 | break; |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | if(rcF == PAGE_IS_IN_RANGE) { |
| 190 | // (*gaps) += 0; |
| 191 | return pdF; |
| 192 | } |
| 193 | |
| 194 | if(rcL == PAGE_IS_IN_RANGE) { |
| 195 | // (*gaps) += 0; |
| 196 | return pdL; |
| 197 | } |
| 198 | |
| 199 | if(rcF == PAGE_IS_IN_THE_FUTURE) { |
| 200 | (*gaps)++; |
| 201 | return pdF; |
| 202 | } |
| 203 | |
| 204 | if(rcL == PAGE_IS_IN_THE_FUTURE) { |
| 205 | (*gaps)++; |
| 206 | return pdL; |
| 207 | } |
| 208 | |
| 209 | // impossible case |
| 210 | (*gaps)++; |
| 211 | return NULL; |
| 212 | } |
| 213 | |
| 214 | static ALWAYS_INLINE_HOT size_t get_page_list_from_pgc(PGC *cache, METRIC *metric, struct rrdengine_instance *ctx, |
| 215 | time_t wanted_start_time_s, time_t wanted_end_time_s, |
| 216 | Pvoid_t *JudyL_page_array, size_t *cache_gaps, |
| 217 | bool open_cache_mode, PDC_PAGE_STATUS tags) { |
| 218 | |
| 219 | size_t pages_found_in_cache = 0; |
| 220 | Word_t metric_id = mrg_metric_id(main_mrg, metric); |
| 221 | |
| 222 | time_t now_s = wanted_start_time_s; |
| 223 | uint32_t dt_s = mrg_metric_get_update_every_s(main_mrg, metric); |
| 224 | |
| 225 | if(!dt_s) |
| 226 | dt_s = nd_profile.update_every; |
| 227 | |
| 228 | time_t previous_page_end_time_s = now_s - dt_s; |
| 229 | bool first = true; |
| 230 | |
| 231 | do { |
| 232 | PGC_PAGE *page = pgc_page_get_and_acquire( |
| 233 | cache, (Word_t)ctx, (Word_t)metric_id, now_s, |
| 234 | (first) ? PGC_SEARCH_CLOSEST : PGC_SEARCH_NEXT); |
| 235 | |
| 236 | first = false; |
| 237 | |
| 238 | if(!page) { |
| 239 | if(previous_page_end_time_s < wanted_end_time_s) |
| 240 | (*cache_gaps)++; |
| 241 | |
| 242 | break; |
| 243 | } |
| 244 | |
| 245 | time_t page_start_time_s = pgc_page_start_time_s(page); |
| 246 | time_t page_end_time_s = pgc_page_end_time_s(page); |
| 247 | uint32_t page_update_every_s = pgc_page_update_every_s(page); |
| 248 | |
| 249 | if(!page_update_every_s) |
| 250 | page_update_every_s = dt_s; |
| 251 | |
| 252 | if(is_page_in_time_range(page_start_time_s, page_end_time_s, wanted_start_time_s, wanted_end_time_s) != PAGE_IS_IN_RANGE) { |
| 253 | // not a useful page for this query |
| 254 | pgc_page_release(cache, page); |
| 255 | page = NULL; |
| 256 | |
| 257 | if(previous_page_end_time_s < wanted_end_time_s) |
| 258 | (*cache_gaps)++; |
| 259 | |
| 260 | break; |
| 261 | } |
| 262 | |
| 263 | if (page_start_time_s - previous_page_end_time_s > (time_t)dt_s) |
| 264 | (*cache_gaps)++; |
| 265 | |
| 266 | Pvoid_t *PValue = PDCJudyLIns(JudyL_page_array, (Word_t) page_start_time_s, PJE0); |
| 267 | if (!PValue || PValue == PJERR) |
| 268 | fatal("DBENGINE: corrupted judy array in %s()", __FUNCTION__ ); |
| 269 | |
| 270 | if (unlikely(*PValue)) |
| 271 | // already exists in our list |
| 272 | pgc_page_release(cache, page); |
| 273 | |
| 274 | else { |
| 275 | |
| 276 | internal_fatal(pgc_page_metric(page) != metric_id, "Wrong metric id in page found in cache"); |
| 277 | internal_fatal(pgc_page_section(page) != (Word_t)ctx, "Wrong section in page found in cache"); |
| 278 | |
| 279 | struct page_details *pd = page_details_get(); |
| 280 | pd->metric_id = metric_id; |
| 281 | pd->first_time_s = page_start_time_s; |
| 282 | pd->last_time_s = page_end_time_s; |
| 283 | pd->update_every_s = page_update_every_s; |
| 284 | pd->page = (open_cache_mode) ? NULL : page; |
| 285 | pd->status |= tags; |
| 286 | |
| 287 | if((pd->page)) { |
| 288 | pd->status |= PDC_PAGE_READY | PDC_PAGE_PRELOADED; |
| 289 | |
| 290 | if(pgd_is_empty(pgc_page_data(page))) |
| 291 | pd->status |= PDC_PAGE_EMPTY; |
| 292 | } |
| 293 | |
| 294 | if(open_cache_mode) { |
| 295 | struct rrdengine_datafile *datafile = pgc_page_data(page); |
| 296 | if(datafile_acquire(datafile, DATAFILE_ACQUIRE_PAGE_DETAILS)) { // for pd |
| 297 | struct extent_io_data *xio = (struct extent_io_data *) pgc_page_custom_data(cache, page); |
| 298 | pd->datafile.ptr = pgc_page_data(page); |
| 299 | pd->datafile.block = xio->block; |
| 300 | pd->datafile.bytes = xio->bytes; |
| 301 | pd->status |= PDC_PAGE_DATAFILE_ACQUIRED | PDC_PAGE_DISK_PENDING; |
| 302 | } |
| 303 | else { |
| 304 | pd->status |= PDC_PAGE_FAILED | PDC_PAGE_FAILED_TO_ACQUIRE_DATAFILE; |
| 305 | } |
| 306 | pgc_page_release(cache, page); |
| 307 | } |
| 308 | |
| 309 | *PValue = pd; |
| 310 | |
| 311 | pages_found_in_cache++; |
| 312 | } |
| 313 | |
| 314 | // prepare for the next iteration |
| 315 | previous_page_end_time_s = page_end_time_s; |
| 316 | |
| 317 | if(page_update_every_s > 0) |
| 318 | dt_s = page_update_every_s; |
| 319 | |
| 320 | // we are going to as for the NEXT page |
| 321 | // so, set this to our first time |
| 322 | now_s = page_start_time_s; |
| 323 | |
| 324 | } while(now_s <= wanted_end_time_s); |
| 325 | |
| 326 | return pages_found_in_cache; |
| 327 | } |
| 328 | |
| 329 | static void pgc_inject_gap(struct rrdengine_instance *ctx, METRIC *metric, time_t start_time_s, time_t end_time_s) { |
| 330 | |
| 331 | time_t db_first_time_s, db_last_time_s; |
| 332 | mrg_metric_get_retention(main_mrg, metric, &db_first_time_s, &db_last_time_s, NULL); |
| 333 | |
| 334 | if(is_page_in_time_range(start_time_s, end_time_s, db_first_time_s, db_last_time_s) != PAGE_IS_IN_RANGE) |
| 335 | return; |
| 336 | |
| 337 | PGC_ENTRY page_entry = { |
| 338 | .hot = false, |
| 339 | .section = (Word_t)ctx, |
| 340 | .metric_id = (Word_t)metric, |
| 341 | .start_time_s = MAX(start_time_s, db_first_time_s), |
| 342 | .end_time_s = MIN(end_time_s, db_last_time_s), |
| 343 | .update_every_s = 0, |
| 344 | .size = 0, |
| 345 | .data = PGD_EMPTY, |
| 346 | }; |
| 347 | |
| 348 | if(page_entry.start_time_s >= page_entry.end_time_s) |
| 349 | return; |
| 350 | |
| 351 | PGC_PAGE *page = pgc_page_add_and_acquire(main_cache, page_entry, NULL); |
| 352 | pgc_page_release(main_cache, page); |
| 353 | } |
| 354 | |
| 355 | static ALWAYS_INLINE_HOT size_t list_has_time_gaps( |
| 356 | struct rrdengine_instance *ctx, |
| 357 | METRIC *metric, |
| 358 | Pvoid_t JudyL_page_array, |
| 359 | time_t wanted_start_time_s, |
| 360 | time_t wanted_end_time_s, |
| 361 | size_t *pages_total, |
| 362 | size_t *pages_found_pass4, |
| 363 | size_t *pages_to_load_from_disk, |
| 364 | size_t *pages_overlapping, |
| 365 | time_t *optimal_end_time_s, |
| 366 | bool populate_gaps, |
| 367 | PDC_PAGE_STATUS *common_status |
| 368 | ) { |
| 369 | // we will recalculate these, so zero them |
| 370 | *pages_to_load_from_disk = 0; |
| 371 | *pages_overlapping = 0; |
| 372 | *optimal_end_time_s = 0; |
| 373 | *common_status = 0; |
| 374 | |
| 375 | bool first; |
| 376 | Pvoid_t *PValue; |
| 377 | Word_t this_page_start_time; |
| 378 | struct page_details *pd; |
| 379 | |
| 380 | size_t gaps = 0; |
| 381 | Word_t metric_id = mrg_metric_id(main_mrg, metric); |
| 382 | |
| 383 | // ------------------------------------------------------------------------ |
| 384 | // PASS 1: remove the preprocessing flags from the pages in PDC |
| 385 | |
| 386 | first = true; |
| 387 | this_page_start_time = 0; |
| 388 | while((PValue = PDCJudyLFirstThenNext(JudyL_page_array, &this_page_start_time, &first))) { |
| 389 | pd = *PValue; |
| 390 | pd->status &= ~(PDC_PAGE_SKIP|PDC_PAGE_PREPROCESSED); |
| 391 | } |
| 392 | |
| 393 | // ------------------------------------------------------------------------ |
| 394 | // PASS 2: emulate processing to find the useful pages |
| 395 | |
| 396 | time_t now_s = wanted_start_time_s; |
| 397 | time_t dt_s = mrg_metric_get_update_every_s(main_mrg, metric); |
| 398 | if(!dt_s) |
| 399 | dt_s = nd_profile.update_every; |
| 400 | |
| 401 | size_t pages_pass2 = 0, pages_pass3 = 0; |
| 402 | while((pd = pdc_find_page_for_time( |
| 403 | JudyL_page_array, now_s, &gaps, |
| 404 | PDC_PAGE_PREPROCESSED, 0))) { |
| 405 | |
| 406 | pd->status |= PDC_PAGE_PREPROCESSED; |
| 407 | pages_pass2++; |
| 408 | |
| 409 | if(pd->update_every_s) |
| 410 | dt_s = pd->update_every_s; |
| 411 | |
| 412 | if(populate_gaps && pd->first_time_s > now_s) |
| 413 | pgc_inject_gap(ctx, metric, now_s, pd->first_time_s); |
| 414 | |
| 415 | now_s = pd->last_time_s + dt_s; |
| 416 | if(now_s > wanted_end_time_s) { |
| 417 | *optimal_end_time_s = pd->last_time_s; |
| 418 | break; |
| 419 | } |
| 420 | } |
| 421 | |
| 422 | if(populate_gaps && now_s < wanted_end_time_s) |
| 423 | pgc_inject_gap(ctx, metric, now_s, wanted_end_time_s); |
| 424 | |
| 425 | // ------------------------------------------------------------------------ |
| 426 | // PASS 3: mark as skipped all the pages not useful |
| 427 | |
| 428 | first = true; |
| 429 | this_page_start_time = 0; |
| 430 | while((PValue = PDCJudyLFirstThenNext(JudyL_page_array, &this_page_start_time, &first))) { |
| 431 | pd = *PValue; |
| 432 | |
| 433 | internal_fatal(pd->metric_id != metric_id, "pd has wrong metric_id"); |
| 434 | |
| 435 | if(!(pd->status & PDC_PAGE_PREPROCESSED)) { |
| 436 | (*pages_overlapping)++; |
| 437 | pd->status |= PDC_PAGE_SKIP; |
| 438 | pd->status &= ~(PDC_PAGE_READY | PDC_PAGE_DISK_PENDING); |
| 439 | *common_status |= pd->status; |
| 440 | continue; |
| 441 | } |
| 442 | |
| 443 | pages_pass3++; |
| 444 | |
| 445 | if(!pd->page) { |
| 446 | pd->page = pgc_page_get_and_acquire(main_cache, (Word_t) ctx, (Word_t) metric_id, pd->first_time_s, PGC_SEARCH_EXACT); |
| 447 | |
| 448 | if(pd->page) { |
| 449 | (*pages_found_pass4)++; |
| 450 | |
| 451 | pd->status &= ~PDC_PAGE_DISK_PENDING; |
| 452 | pd->status |= PDC_PAGE_READY | PDC_PAGE_PRELOADED | PDC_PAGE_PRELOADED_PASS4; |
| 453 | |
| 454 | if(pgd_is_empty(pgc_page_data(pd->page))) |
| 455 | pd->status |= PDC_PAGE_EMPTY; |
| 456 | |
| 457 | } |
| 458 | else if(!(pd->status & PDC_PAGE_FAILED) && (pd->status & PDC_PAGE_DATAFILE_ACQUIRED)) { |
| 459 | (*pages_to_load_from_disk)++; |
| 460 | |
| 461 | pd->status |= PDC_PAGE_DISK_PENDING; |
| 462 | |
| 463 | internal_fatal(pd->status & PDC_PAGE_SKIP, "page is disk pending and skipped"); |
| 464 | internal_fatal(!pd->datafile.ptr, "datafile is NULL"); |
| 465 | internal_fatal(!pd->datafile.bytes, "datafile.bytes zero"); |
| 466 | internal_fatal(!pd->datafile.block, "datafile.block is zero"); |
| 467 | } |
| 468 | } |
| 469 | else { |
| 470 | pd->status &= ~PDC_PAGE_DISK_PENDING; |
| 471 | pd->status |= (PDC_PAGE_READY | PDC_PAGE_PRELOADED); |
| 472 | } |
| 473 | |
| 474 | *common_status |= pd->status; |
| 475 | } |
| 476 | |
| 477 | internal_fatal(pages_pass2 != pages_pass3, |
| 478 | "DBENGINE: page count does not match"); |
| 479 | |
| 480 | *pages_total = pages_pass2; |
| 481 | |
| 482 | return gaps; |
| 483 | } |
| 484 | |
| 485 | // ---------------------------------------------------------------------------- |
| 486 | |
| 487 | typedef void (*page_found_callback_t)(PGC_PAGE *page, void *data); |
| 488 | |
| 489 | static NOT_INLINE_HOT size_t get_page_list_from_journal_v2(struct rrdengine_instance *ctx, METRIC *metric, |
| 490 | usec_t start_time_ut, usec_t end_time_ut, |
| 491 | page_found_callback_t callback, void *callback_data) |
| 492 | { |
| 493 | nd_uuid_t *uuid = mrg_metric_uuid(main_mrg, metric); |
| 494 | if (unlikely(!uuid)) |
| 495 | return 0; |
| 496 | |
| 497 | Word_t metric_id = mrg_metric_id(main_mrg, metric); |
| 498 | |
| 499 | time_t wanted_start_time_s = (time_t)(start_time_ut / USEC_PER_SEC); |
| 500 | time_t wanted_end_time_s = (time_t)(end_time_ut / USEC_PER_SEC); |
| 501 | |
| 502 | size_t pages_found = 0; |
| 503 | |
| 504 | NJFV2IDX_FIND_STATE state = { |
| 505 | .init = false, |
| 506 | .last = 0, |
| 507 | .ctx = ctx, |
| 508 | .wanted_start_time_s = wanted_start_time_s, |
| 509 | .wanted_end_time_s = wanted_end_time_s, |
| 510 | .j2_header_acquired = NULL, |
| 511 | }; |
| 512 | |
| 513 | struct rrdengine_datafile *datafile; |
| 514 | while((datafile = njfv2idx_find_and_acquire_j2_header(&state))) { |
| 515 | struct journal_v2_header *j2_header = state.j2_header_acquired; |
| 516 | |
| 517 | if (unlikely(!j2_header)) { |
| 518 | datafile_release(datafile, DATAFILE_ACQUIRE_PAGE_DETAILS); |
| 519 | continue; |
| 520 | } |
| 521 | |
| 522 | char file_path[RRDENG_PATH_MAX]; |
| 523 | journalfile_v2_generate_path(datafile, file_path, sizeof(file_path)); |
| 524 | PROTECTED_ACCESS_SETUP(datafile->journalfile->mmap.data, datafile->journalfile->mmap.size, file_path, "read"); |
| 525 | if(no_signal_received) { |
| 526 | time_t journal_start_time_s = (time_t)(j2_header->start_time_ut / USEC_PER_SEC); |
| 527 | size_t journal_v2_file_size = datafile->journalfile->mmap.size; |
| 528 | |
| 529 | // the datafile possibly contains useful data for this query |
| 530 | |
| 531 | size_t journal_metric_count = (size_t)j2_header->metric_count; |
| 532 | struct journal_metric_list *uuid_list = |
| 533 | (struct journal_metric_list *)((uint8_t *)j2_header + j2_header->metric_offset); |
| 534 | size_t metric_offset = (uint8_t *)uuid_list - (uint8_t *)j2_header; |
| 535 | |
| 536 | size_t metric_list_size; |
| 537 | if (__builtin_mul_overflow(journal_metric_count, sizeof(*uuid_list), &metric_list_size) || |
| 538 | metric_offset > journal_v2_file_size || |
| 539 | metric_list_size > journal_v2_file_size - metric_offset) { |
| 540 | nd_log_limit_static_thread_var(erl, 60, 0); |
| 541 | nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, |
| 542 | "DBENGINE: Metric list exceeds journal file size in journalfile %u of tier %u (metric_offset=%zu, list_size=%zu, file_size=%zu)", |
| 543 | datafile->fileno, datafile->tier, metric_offset, metric_list_size, journal_v2_file_size); |
| 544 | goto release_journal; |
| 545 | } |
| 546 | |
| 547 | struct journal_metric_list *uuid_entry = |
| 548 | bsearch(uuid, uuid_list, journal_metric_count, sizeof(*uuid_list), journal_metric_uuid_compare); |
| 549 | |
| 550 | if (unlikely(!uuid_entry)) { |
| 551 | // our UUID is not in this datafile |
| 552 | goto release_journal; |
| 553 | } |
| 554 | |
| 555 | struct journal_page_header *page_list_header = |
| 556 | (struct journal_page_header *)((uint8_t *)j2_header + uuid_entry->page_offset); |
| 557 | size_t page_offset = (uint8_t *)page_list_header - (uint8_t *)j2_header; |
| 558 | if (page_offset > journal_v2_file_size - sizeof(*page_list_header)) { |
| 559 | nd_log_limit_static_thread_var(erl, 60, 0); |
| 560 | nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, |
| 561 | "DBENGINE: Invalid page list header in journalfile %u of tier %u", |
| 562 | datafile->fileno, datafile->tier); |
| 563 | goto release_journal; |
| 564 | } |
| 565 | |
| 566 | struct journal_page_list *page_list = |
| 567 | (struct journal_page_list *)((uint8_t *)page_list_header + sizeof(*page_list_header)); |
| 568 | struct journal_extent_list *extent_list = (void *)((uint8_t *)j2_header + j2_header->extent_offset); |
| 569 | uint32_t extent_entries = j2_header->extent_count; |
| 570 | uint32_t uuid_page_entries = page_list_header->entries; |
| 571 | size_t page_list_size; |
| 572 | |
| 573 | if (__builtin_mul_overflow((size_t)uuid_page_entries, sizeof(*page_list), &page_list_size) || |
| 574 | page_list_size > journal_v2_file_size - page_offset - sizeof(*page_list_header)) { |
| 575 | nd_log_limit_static_thread_var(erl, 60, 0); |
| 576 | nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, |
| 577 | "DBENGINE: Page list exceeds journal file size in journalfile %u of tier %u", |
| 578 | datafile->fileno, datafile->tier); |
| 579 | goto release_journal; |
| 580 | } |
| 581 | |
| 582 | for (uint32_t index = 0; index < uuid_page_entries; index++) { |
| 583 | struct journal_page_list *page_entry_in_journal = &page_list[index]; |
| 584 | |
| 585 | time_t page_first_time_s = page_entry_in_journal->delta_start_s + journal_start_time_s; |
| 586 | time_t page_last_time_s = page_entry_in_journal->delta_end_s + journal_start_time_s; |
| 587 | |
| 588 | TIME_RANGE_COMPARE prc = |
| 589 | is_page_in_time_range(page_first_time_s, page_last_time_s, wanted_start_time_s, wanted_end_time_s); |
| 590 | |
| 591 | if (prc == PAGE_IS_IN_THE_PAST) |
| 592 | continue; |
| 593 | |
| 594 | if (prc == PAGE_IS_IN_THE_FUTURE) |
| 595 | break; |
| 596 | |
| 597 | // Make sure index is valid for this file |
| 598 | if (page_entry_in_journal->extent_index >= extent_entries) { |
| 599 | nd_log_limit_static_thread_var(erl, 60, 0); |
| 600 | nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, |
| 601 | "DBENGINE: Invalid extent index in journalfile %u", |
| 602 | datafile->fileno); |
| 603 | break; |
| 604 | } |
| 605 | |
| 606 | uint32_t page_update_every_s = page_entry_in_journal->update_every_s; |
| 607 | |
| 608 | if (datafile_acquire(datafile, DATAFILE_ACQUIRE_OPEN_CACHE)) { |
| 609 | //for open cache item |
| 610 | // add this page to open cache |
| 611 | bool added = false; |
| 612 | struct extent_io_data ei = {0}; |
| 613 | ei.block = OFFSET_TO_BLOCK(extent_list[page_entry_in_journal->extent_index].datafile_offset); |
| 614 | ei.bytes = extent_list[page_entry_in_journal->extent_index].datafile_size; |
| 615 | ei.fileno = datafile->fileno; |
| 616 | |
| 617 | PGC_ENTRY e = {0}; |
| 618 | e.hot = false; |
| 619 | e.section = (Word_t)ctx; |
| 620 | e.metric_id = metric_id; |
| 621 | e.start_time_s = page_first_time_s; |
| 622 | e.end_time_s = page_last_time_s; |
| 623 | e.update_every_s = page_update_every_s; |
| 624 | e.data = datafile; |
| 625 | e.size = 0; |
| 626 | e.custom_data = (uint8_t *)&ei; |
| 627 | PGC_PAGE *page = pgc_page_add_and_acquire(open_cache, e, &added); |
| 628 | |
| 629 | if (!added) |
| 630 | datafile_release(datafile, DATAFILE_ACQUIRE_OPEN_CACHE); |
| 631 | |
| 632 | callback(page, callback_data); |
| 633 | |
| 634 | pgc_page_release(open_cache, page); |
| 635 | |
| 636 | pages_found++; |
| 637 | } |
| 638 | } |
| 639 | } |
| 640 | else { |
| 641 | nd_log_limit_static_thread_var(erl, 10, 0); |
| 642 | nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, "DBENGINE: failed to access journal file %u of tier %d (SIGBUS)", |
| 643 | datafile->fileno, datafile->ctx->config.tier); |
| 644 | } |
| 645 | |
| 646 | release_journal: |
| 647 | journalfile_v2_data_release(datafile->journalfile); |
| 648 | datafile_release(datafile, DATAFILE_ACQUIRE_PAGE_DETAILS); |
| 649 | } |
| 650 | |
| 651 | return pages_found; |
| 652 | } |
| 653 | |
| 654 | void add_page_details_from_journal_v2(PGC_PAGE *page, void *JudyL_pptr) { |
| 655 | struct rrdengine_datafile *datafile = pgc_page_data(page); |
| 656 | |
| 657 | if(!datafile_acquire(datafile, DATAFILE_ACQUIRE_PAGE_DETAILS)) // for pd |
| 658 | return; |
| 659 | |
| 660 | Pvoid_t *PValue = PDCJudyLIns(JudyL_pptr, pgc_page_start_time_s(page), PJE0); |
| 661 | if (!PValue || PValue == PJERR) |
| 662 | fatal("DBENGINE: corrupted judy array"); |
| 663 | |
| 664 | if (unlikely(*PValue)) { |
| 665 | datafile_release(datafile, DATAFILE_ACQUIRE_PAGE_DETAILS); |
| 666 | return; |
| 667 | } |
| 668 | |
| 669 | Word_t metric_id = pgc_page_metric(page); |
| 670 | |
| 671 | // let's add it to the judy |
| 672 | struct extent_io_data *ei = pgc_page_custom_data(open_cache, page); |
| 673 | struct page_details *pd = page_details_get(); |
| 674 | *PValue = pd; |
| 675 | |
| 676 | pd->datafile.block = ei->block; |
| 677 | pd->datafile.bytes = ei->bytes; |
| 678 | pd->first_time_s = pgc_page_start_time_s(page); |
| 679 | pd->last_time_s = pgc_page_end_time_s(page); |
| 680 | pd->datafile.ptr = datafile; |
| 681 | pd->update_every_s = (uint32_t) pgc_page_update_every_s(page); |
| 682 | pd->metric_id = metric_id; |
| 683 | pd->status |= PDC_PAGE_DISK_PENDING | PDC_PAGE_SOURCE_JOURNAL_V2 | PDC_PAGE_DATAFILE_ACQUIRED; |
| 684 | } |
| 685 | |
| 686 | // Return a judyL will all pages that have start_time_ut and end_time_ut |
| 687 | // Pvalue of the judy will be the end time for that page |
| 688 | // DBENGINE2: |
| 689 | #define time_delta(finish, pass) do { if(pass) { usec_t t = pass; (pass) = (finish) - (pass); (finish) = t; } } while(0) |
| 690 | static ALWAYS_INLINE_HOT Pvoid_t get_page_list( |
| 691 | struct rrdengine_instance *ctx, |
| 692 | METRIC *metric, |
| 693 | usec_t start_time_ut, |
| 694 | usec_t end_time_ut, |
| 695 | time_t *optimal_end_time_s, |
| 696 | size_t *pages_to_load_from_disk, |
| 697 | PDC_PAGE_STATUS *common_status |
| 698 | ) { |
| 699 | *optimal_end_time_s = 0; |
| 700 | *pages_to_load_from_disk = 0; |
| 701 | *common_status = 0; |
| 702 | |
| 703 | Pvoid_t JudyL_page_array = (Pvoid_t) NULL; |
| 704 | |
| 705 | time_t wanted_start_time_s = (time_t)(start_time_ut / USEC_PER_SEC); |
| 706 | time_t wanted_end_time_s = (time_t)(end_time_ut / USEC_PER_SEC); |
| 707 | |
| 708 | size_t pages_found_in_main_cache = 0, |
| 709 | pages_found_in_open_cache = 0, |
| 710 | pages_found_in_journals_v2 = 0, |
| 711 | pages_found_pass4 = 0, |
| 712 | pages_overlapping = 0, |
| 713 | pages_total = 0; |
| 714 | |
| 715 | size_t cache_gaps = 0, query_gaps = 0; |
| 716 | bool done_v2 = false, done_open = false, done_pass4 = false; |
| 717 | |
| 718 | usec_t pass1_ut = 0, pass2_ut = 0, pass3_ut = 0, pass4_ut = 0, finish_ut = 0; |
| 719 | |
| 720 | // -------------------------------------------------------------- |
| 721 | // PASS 1: Check what the main page cache has available |
| 722 | |
| 723 | pass1_ut = now_monotonic_usec(); |
| 724 | size_t pages_pass1 = get_page_list_from_pgc(main_cache, metric, ctx, wanted_start_time_s, wanted_end_time_s, |
| 725 | &JudyL_page_array, &cache_gaps, |
| 726 | false, PDC_PAGE_SOURCE_MAIN_CACHE); |
| 727 | query_gaps += cache_gaps; |
| 728 | pages_found_in_main_cache += pages_pass1; |
| 729 | pages_total += pages_pass1; |
| 730 | |
| 731 | if(pages_found_in_main_cache && !cache_gaps) { |
| 732 | query_gaps = list_has_time_gaps(ctx, metric, JudyL_page_array, wanted_start_time_s, wanted_end_time_s, |
| 733 | &pages_total, &pages_found_pass4, pages_to_load_from_disk, &pages_overlapping, |
| 734 | optimal_end_time_s, false, common_status); |
| 735 | |
| 736 | if (pages_total && !query_gaps) |
| 737 | goto we_are_done; |
| 738 | } |
| 739 | |
| 740 | // -------------------------------------------------------------- |
| 741 | // PASS 2: Check what the open journal page cache has available |
| 742 | // these will be loaded from disk |
| 743 | |
| 744 | pass2_ut = now_monotonic_usec(); |
| 745 | size_t pages_pass2 = get_page_list_from_pgc(open_cache, metric, ctx, wanted_start_time_s, wanted_end_time_s, |
| 746 | &JudyL_page_array, &cache_gaps, |
| 747 | true, PDC_PAGE_SOURCE_OPEN_CACHE); |
| 748 | query_gaps += cache_gaps; |
| 749 | pages_found_in_open_cache += pages_pass2; |
| 750 | pages_total += pages_pass2; |
| 751 | done_open = true; |
| 752 | |
| 753 | if(pages_found_in_open_cache) { |
| 754 | query_gaps = list_has_time_gaps(ctx, metric, JudyL_page_array, wanted_start_time_s, wanted_end_time_s, |
| 755 | &pages_total, &pages_found_pass4, pages_to_load_from_disk, &pages_overlapping, |
| 756 | optimal_end_time_s, false, common_status); |
| 757 | |
| 758 | if (pages_total && !query_gaps) |
| 759 | goto we_are_done; |
| 760 | } |
| 761 | |
| 762 | // -------------------------------------------------------------- |
| 763 | // PASS 3: Check Journal v2 to fill the gaps |
| 764 | |
| 765 | pass3_ut = now_monotonic_usec(); |
| 766 | size_t pages_pass3 = get_page_list_from_journal_v2(ctx, metric, start_time_ut, end_time_ut, |
| 767 | add_page_details_from_journal_v2, &JudyL_page_array); |
| 768 | pages_found_in_journals_v2 += pages_pass3; |
| 769 | pages_total += pages_pass3; |
| 770 | done_v2 = true; |
| 771 | |
| 772 | // -------------------------------------------------------------- |
| 773 | // PASS 4: Check the cache again |
| 774 | // and calculate the time gaps in the query |
| 775 | // THIS IS REQUIRED AFTER JOURNAL V2 LOOKUP |
| 776 | |
| 777 | pass4_ut = now_monotonic_usec(); |
| 778 | query_gaps = list_has_time_gaps(ctx, metric, JudyL_page_array, wanted_start_time_s, wanted_end_time_s, |
| 779 | &pages_total, &pages_found_pass4, pages_to_load_from_disk, &pages_overlapping, |
| 780 | optimal_end_time_s, true, common_status); |
| 781 | done_pass4 = true; |
| 782 | |
| 783 | we_are_done: |
| 784 | finish_ut = now_monotonic_usec(); |
| 785 | time_delta(finish_ut, pass4_ut); // do not change the order |
| 786 | time_delta(finish_ut, pass3_ut); // do not change the order |
| 787 | time_delta(finish_ut, pass2_ut); // do not change the order |
| 788 | time_delta(finish_ut, pass1_ut); // do not change the order |
| 789 | |
| 790 | time_and_count_add(&rrdeng_cache_efficiency_stats.prep_time_in_main_cache_lookup, pass1_ut); |
| 791 | |
| 792 | if(done_open) { |
| 793 | time_and_count_add(&rrdeng_cache_efficiency_stats.prep_time_in_open_cache_lookup, pass2_ut); |
| 794 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_meta_source_open_cache, pages_found_in_open_cache, __ATOMIC_RELAXED); |
| 795 | } |
| 796 | |
| 797 | if(done_v2) { |
| 798 | time_and_count_add(&rrdeng_cache_efficiency_stats.prep_time_in_journal_v2_lookup, pass3_ut); |
| 799 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_meta_source_journal_v2, pages_found_in_journals_v2, __ATOMIC_RELAXED); |
| 800 | } |
| 801 | |
| 802 | if(done_pass4) { |
| 803 | time_and_count_add(&rrdeng_cache_efficiency_stats.prep_time_in_pass4_lookup, pass4_ut); |
| 804 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_data_source_main_cache_at_pass4, pages_found_pass4, __ATOMIC_RELAXED); |
| 805 | } |
| 806 | |
| 807 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.queries_planned_with_gaps, (query_gaps) ? 1 : 0, __ATOMIC_RELAXED); |
| 808 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_total, pages_total, __ATOMIC_RELAXED); |
| 809 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_meta_source_main_cache, pages_found_in_main_cache, __ATOMIC_RELAXED); |
| 810 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_data_source_main_cache, pages_found_in_main_cache, __ATOMIC_RELAXED); |
| 811 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_to_load_from_disk, *pages_to_load_from_disk, __ATOMIC_RELAXED); |
| 812 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_overlapping_skipped, pages_overlapping, __ATOMIC_RELAXED); |
| 813 | |
| 814 | return JudyL_page_array; |
| 815 | } |
| 816 | |
| 817 | ALWAYS_INLINE void rrdeng_prep_wait(PDC *pdc) { |
| 818 | if (unlikely(pdc && !pdc->prep_done)) { |
| 819 | usec_t started_ut = now_monotonic_usec(); |
| 820 | completion_wait_for(&pdc->prep_completion); |
| 821 | pdc->prep_done = true; |
| 822 | time_and_count_add(&rrdeng_cache_efficiency_stats.query_time_wait_for_prep, now_monotonic_usec() - started_ut); |
| 823 | } |
| 824 | } |
| 825 | |
| 826 | ALWAYS_INLINE_HOT void rrdeng_prep_query(struct page_details_control *pdc, bool worker) { |
| 827 | if(worker) |
| 828 | worker_is_busy(UV_EVENT_DBENGINE_QUERY); |
| 829 | |
| 830 | pdc->page_list_JudyL = get_page_list(pdc->ctx, pdc->metric, |
| 831 | pdc->start_time_s * USEC_PER_SEC, |
| 832 | pdc->end_time_s * USEC_PER_SEC, |
| 833 | &pdc->optimal_end_time_s, |
| 834 | &pdc->pages_to_load_from_disk, |
| 835 | &pdc->common_status); |
| 836 | |
| 837 | internal_fatal(pdc->pages_to_load_from_disk && !(pdc->common_status & PDC_PAGE_DISK_PENDING), |
| 838 | "DBENGINE: PDC reports there are %zu pages to load from disk, " |
| 839 | "but none of the pages has the PDC_PAGE_DISK_PENDING flag", |
| 840 | pdc->pages_to_load_from_disk); |
| 841 | |
| 842 | internal_fatal(!pdc->pages_to_load_from_disk && (pdc->common_status & PDC_PAGE_DISK_PENDING), |
| 843 | "DBENGINE: PDC reports there are no pages to load from disk, " |
| 844 | "but one or more pages have the PDC_PAGE_DISK_PENDING flag"); |
| 845 | |
| 846 | if (pdc->pages_to_load_from_disk && pdc->page_list_JudyL) { |
| 847 | pdc_acquire(pdc); // we get 1 for the 1st worker in the chain: do_read_page_list_work() |
| 848 | usec_t start_ut = now_monotonic_usec(); |
| 849 | if(likely(pdc->priority == STORAGE_PRIORITY_SYNCHRONOUS)) { |
| 850 | pdc_route_synchronously(pdc->ctx, pdc); |
| 851 | time_and_count_add(&rrdeng_cache_efficiency_stats.prep_time_to_route_sync, now_monotonic_usec() - start_ut); |
| 852 | } |
| 853 | else if(likely(pdc->priority == STORAGE_PRIORITY_SYNCHRONOUS_FIRST)) { |
| 854 | pdc_route_synchronously_first(pdc->ctx, pdc); |
| 855 | time_and_count_add(&rrdeng_cache_efficiency_stats.prep_time_to_route_syncfirst, now_monotonic_usec() - start_ut); |
| 856 | } |
| 857 | else { |
| 858 | pdc_route_asynchronously(pdc->ctx, pdc); |
| 859 | time_and_count_add(&rrdeng_cache_efficiency_stats.prep_time_to_route_async, now_monotonic_usec() - start_ut); |
| 860 | } |
| 861 | } |
| 862 | else |
| 863 | completion_mark_complete(&pdc->page_completion); |
| 864 | |
| 865 | completion_mark_complete(&pdc->prep_completion); |
| 866 | |
| 867 | pdc_release_and_destroy_if_unreferenced(pdc, true, true); |
| 868 | |
| 869 | if(worker) |
| 870 | worker_is_idle(); |
| 871 | } |
| 872 | |
| 873 | /** |
| 874 | * Searches for pages in a time range and triggers disk I/O if necessary and possible. |
| 875 | * @param ctx DB context |
| 876 | * @param handle query handle as initialized |
| 877 | * @param start_time_ut inclusive starting time in usec |
| 878 | * @param end_time_ut inclusive ending time in usec |
| 879 | * @return 1 / 0 (pages found or not found) |
| 880 | */ |
| 881 | ALWAYS_INLINE_HOT void pg_cache_preload(struct rrdeng_query_handle *handle) { |
| 882 | if (unlikely(!handle || !handle->metric)) |
| 883 | return; |
| 884 | |
| 885 | __atomic_add_fetch(&handle->ctx->atomic.inflight_queries, 1, __ATOMIC_RELAXED); |
| 886 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.currently_running_queries, 1, __ATOMIC_RELAXED); |
| 887 | handle->pdc = pdc_get(); |
| 888 | handle->pdc->ctx = handle->ctx; |
| 889 | handle->pdc->refcount = 1; |
| 890 | spinlock_init(&handle->pdc->refcount_spinlock); |
| 891 | handle->pdc->metric = mrg_metric_dup(main_mrg, handle->metric); |
| 892 | if(!handle->pdc->metric) { |
| 893 | // metric has been deleted, mark completions and return |
| 894 | completion_init(&handle->pdc->prep_completion); |
| 895 | completion_init(&handle->pdc->page_completion); |
| 896 | completion_mark_complete(&handle->pdc->prep_completion); |
| 897 | completion_mark_complete(&handle->pdc->page_completion); |
| 898 | pdc_release_and_destroy_if_unreferenced(handle->pdc, true, true); |
| 899 | handle->pdc = NULL; |
| 900 | __atomic_sub_fetch(&handle->ctx->atomic.inflight_queries, 1, __ATOMIC_RELAXED); |
| 901 | __atomic_sub_fetch(&rrdeng_cache_efficiency_stats.currently_running_queries, 1, __ATOMIC_RELAXED); |
| 902 | return; |
| 903 | } |
| 904 | handle->pdc->start_time_s = handle->start_time_s; |
| 905 | handle->pdc->end_time_s = handle->end_time_s; |
| 906 | handle->pdc->priority = handle->priority; |
| 907 | handle->pdc->optimal_end_time_s = handle->end_time_s; |
| 908 | completion_init(&handle->pdc->prep_completion); |
| 909 | completion_init(&handle->pdc->page_completion); |
| 910 | |
| 911 | if(ctx_is_available_for_queries(handle->ctx)) { |
| 912 | handle->pdc->refcount++; // we get 1 for the query thread and 1 for the prep thread |
| 913 | |
| 914 | if(unlikely(handle->pdc->priority == STORAGE_PRIORITY_SYNCHRONOUS || handle->pdc->priority == STORAGE_PRIORITY_SYNCHRONOUS_FIRST)) |
| 915 | rrdeng_prep_query(handle->pdc, false); |
| 916 | else |
| 917 | rrdeng_enq_cmd(handle->ctx, RRDENG_OPCODE_QUERY, handle->pdc, NULL, handle->priority, NULL, NULL); |
| 918 | } |
| 919 | else { |
| 920 | completion_mark_complete(&handle->pdc->prep_completion); |
| 921 | completion_mark_complete(&handle->pdc->page_completion); |
| 922 | } |
| 923 | } |
| 924 | |
| 925 | /* |
| 926 | * Searches for the first page between start_time and end_time and gets a reference. |
| 927 | * start_time and end_time are inclusive. |
| 928 | * If index is NULL lookup by UUID (id). |
| 929 | */ |
| 930 | struct pgc_page *pg_cache_lookup_next( |
| 931 | struct rrdengine_instance *ctx, |
| 932 | PDC *pdc, |
| 933 | time_t now_s, |
| 934 | uint32_t last_update_every_s, |
| 935 | size_t *entries |
| 936 | ) { |
| 937 | if (unlikely(!pdc)) |
| 938 | return NULL; |
| 939 | |
| 940 | rrdeng_prep_wait(pdc); |
| 941 | |
| 942 | if (unlikely(!pdc->page_list_JudyL)) |
| 943 | return NULL; |
| 944 | |
| 945 | usec_t start_ut = now_monotonic_usec(); |
| 946 | size_t gaps = 0; |
| 947 | bool waited = false, preloaded; |
| 948 | PGC_PAGE *page = NULL; |
| 949 | |
| 950 | while(!page) { |
| 951 | bool page_from_pd = false; |
| 952 | preloaded = false; |
| 953 | struct page_details *pd = pdc_find_page_for_time( |
| 954 | pdc->page_list_JudyL, now_s, &gaps, |
| 955 | PDC_PAGE_PROCESSED, PDC_PAGE_EMPTY); |
| 956 | |
| 957 | if (!pd) |
| 958 | break; |
| 959 | |
| 960 | page = pd->page; |
| 961 | page_from_pd = true; |
| 962 | preloaded = pdc_page_status_check(pd, PDC_PAGE_PRELOADED); |
| 963 | if(!page) { |
| 964 | if(!completion_is_done(&pdc->page_completion)) { |
| 965 | page = pgc_page_get_and_acquire(main_cache, (Word_t)ctx, |
| 966 | pd->metric_id, pd->first_time_s, PGC_SEARCH_EXACT); |
| 967 | page_from_pd = false; |
| 968 | preloaded = pdc_page_status_check(pd, PDC_PAGE_PRELOADED); |
| 969 | } |
| 970 | |
| 971 | if(!page) { |
| 972 | pdc->completed_jobs = |
| 973 | completion_wait_for_a_job(&pdc->page_completion, pdc->completed_jobs); |
| 974 | |
| 975 | page = pd->page; |
| 976 | page_from_pd = true; |
| 977 | preloaded = pdc_page_status_check(pd, PDC_PAGE_PRELOADED); |
| 978 | waited = true; |
| 979 | } |
| 980 | } |
| 981 | |
| 982 | if(page && pgd_is_empty(pgc_page_data(page))) |
| 983 | pdc_page_status_set(pd, PDC_PAGE_EMPTY); |
| 984 | |
| 985 | if(!page || pdc_page_status_check(pd, PDC_PAGE_QUERY_GLOBAL_SKIP_LIST | PDC_PAGE_EMPTY)) { |
| 986 | page = NULL; |
| 987 | continue; |
| 988 | } |
| 989 | |
| 990 | // we now have page and is not empty |
| 991 | |
| 992 | time_t page_start_time_s = pgc_page_start_time_s(page); |
| 993 | time_t page_end_time_s = pgc_page_end_time_s(page); |
| 994 | uint32_t page_update_every_s = pgc_page_update_every_s(page); |
| 995 | |
| 996 | if(unlikely(page_start_time_s == INVALID_TIME || page_end_time_s == INVALID_TIME)) { |
| 997 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_zero_time_skipped, 1, __ATOMIC_RELAXED); |
| 998 | pgc_page_to_clean_evict_or_release(main_cache, page); |
| 999 | pdc_page_status_set(pd, PDC_PAGE_INVALID | PDC_PAGE_RELEASED); |
| 1000 | pd->page = page = NULL; |
| 1001 | continue; |
| 1002 | } |
| 1003 | else { |
| 1004 | if (unlikely(page_update_every_s <= 0 || page_update_every_s > 86400)) { |
| 1005 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_invalid_update_every_fixed, 1, __ATOMIC_RELAXED); |
| 1006 | page_update_every_s = pgc_page_fix_update_every(page, last_update_every_s); |
| 1007 | pd->update_every_s = page_update_every_s; |
| 1008 | } |
| 1009 | |
| 1010 | size_t entries_by_size = pgd_slots_used(pgc_page_data(page)); |
| 1011 | size_t entries_by_time = page_entries_by_time(page_start_time_s, page_end_time_s, page_update_every_s); |
| 1012 | if(unlikely(entries_by_size < entries_by_time)) { |
| 1013 | time_t fixed_page_end_time_s = (time_t)(page_start_time_s + (entries_by_size - 1) * page_update_every_s); |
| 1014 | pd->last_time_s = page_end_time_s = pgc_page_fix_end_time_s(page, fixed_page_end_time_s); |
| 1015 | entries_by_time = (page_end_time_s - (page_start_time_s - page_update_every_s)) / page_update_every_s; |
| 1016 | |
| 1017 | internal_fatal(entries_by_size != entries_by_time, "DBENGINE: wrong entries by time again!"); |
| 1018 | |
| 1019 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_invalid_entries_fixed, 1, __ATOMIC_RELAXED); |
| 1020 | } |
| 1021 | *entries = entries_by_time; |
| 1022 | } |
| 1023 | |
| 1024 | if(unlikely(page_end_time_s < now_s)) { |
| 1025 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_past_time_skipped, 1, __ATOMIC_RELAXED); |
| 1026 | pgc_page_release(main_cache, page); |
| 1027 | pdc_page_status_set(pd, PDC_PAGE_SKIP | PDC_PAGE_RELEASED); |
| 1028 | pd->page = page = NULL; |
| 1029 | continue; |
| 1030 | } |
| 1031 | |
| 1032 | if(page_from_pd) |
| 1033 | // PDC_PAGE_RELEASED is for pdc_destroy() to not release the page twice - the caller will release it |
| 1034 | pdc_page_status_set(pd, PDC_PAGE_RELEASED | PDC_PAGE_PROCESSED); |
| 1035 | else |
| 1036 | pdc_page_status_set(pd, PDC_PAGE_PROCESSED); |
| 1037 | } |
| 1038 | |
| 1039 | if(gaps && !pdc->executed_with_gaps) |
| 1040 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.queries_executed_with_gaps, 1, __ATOMIC_RELAXED); |
| 1041 | pdc->executed_with_gaps = +gaps; |
| 1042 | |
| 1043 | if(page) { |
| 1044 | if(waited) |
| 1045 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.page_next_wait_loaded, 1, __ATOMIC_RELAXED); |
| 1046 | else |
| 1047 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.page_next_nowait_loaded, 1, __ATOMIC_RELAXED); |
| 1048 | } |
| 1049 | else { |
| 1050 | if(waited) |
| 1051 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.page_next_wait_failed, 1, __ATOMIC_RELAXED); |
| 1052 | else |
| 1053 | __atomic_add_fetch(&rrdeng_cache_efficiency_stats.page_next_nowait_failed, 1, __ATOMIC_RELAXED); |
| 1054 | } |
| 1055 | |
| 1056 | if(waited) { |
| 1057 | if(preloaded) |
| 1058 | time_and_count_add(&rrdeng_cache_efficiency_stats.query_time_to_slow_preload_next_page, now_monotonic_usec() - start_ut); |
| 1059 | else |
| 1060 | time_and_count_add(&rrdeng_cache_efficiency_stats.query_time_to_slow_disk_next_page, now_monotonic_usec() - start_ut); |
| 1061 | } |
| 1062 | else { |
| 1063 | if(preloaded) |
| 1064 | time_and_count_add(&rrdeng_cache_efficiency_stats.query_time_to_fast_preload_next_page, now_monotonic_usec() - start_ut); |
| 1065 | else |
| 1066 | time_and_count_add(&rrdeng_cache_efficiency_stats.query_time_to_fast_disk_next_page, now_monotonic_usec() - start_ut); |
| 1067 | } |
| 1068 | |
| 1069 | return page; |
| 1070 | } |
| 1071 | |
| 1072 | void pgc_open_add_hot_page( |
| 1073 | Word_t section, |
| 1074 | Word_t metric_id, |
| 1075 | time_t start_time_s, |
| 1076 | time_t end_time_s, |
| 1077 | uint32_t update_every_s, |
| 1078 | struct rrdengine_datafile *datafile, |
| 1079 | uint64_t extent_offset, |
| 1080 | unsigned extent_size) |
| 1081 | { |
| 1082 | if (unlikely(!rrdeng_valid_extent_disk_size(extent_size))) { |
| 1083 | nd_log_limit_static_thread_var(erl, 10, 0); |
| 1084 | nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, |
| 1085 | "DBENGINE: skipped adding open-cache page for datafile %u of tier %u, " |
| 1086 | "extent at offset %" PRIu64 " has invalid size %u", |
| 1087 | datafile->fileno, datafile->tier, extent_offset, extent_size); |
| 1088 | return; |
| 1089 | } |
| 1090 | |
| 1091 | if(!datafile_acquire(datafile, DATAFILE_ACQUIRE_OPEN_CACHE)) { // for open cache item |
| 1092 | nd_log_limit_static_thread_var(erl, 10, 0); |
| 1093 | nd_log_limit(&erl, NDLS_DAEMON, NDLP_INFO, |
| 1094 | "DBENGINE: skipped adding open-cache page for datafile %u of tier %u (deletion in progress)", |
| 1095 | datafile->fileno, datafile->tier); |
| 1096 | return; |
| 1097 | } |
| 1098 | |
| 1099 | struct extent_io_data ext_io_data = { |
| 1100 | .fileno = datafile->fileno, |
| 1101 | .block = OFFSET_TO_BLOCK(extent_offset), |
| 1102 | .bytes = extent_size, |
| 1103 | }; |
| 1104 | |
| 1105 | PGC_ENTRY page_entry = { |
| 1106 | .hot = true, |
| 1107 | .section = section, |
| 1108 | .metric_id = metric_id, |
| 1109 | .start_time_s = start_time_s, |
| 1110 | .end_time_s = end_time_s, |
| 1111 | .update_every_s = update_every_s, |
| 1112 | .size = 0, |
| 1113 | .data = datafile, |
| 1114 | .custom_data = (uint8_t *) &ext_io_data, |
| 1115 | }; |
| 1116 | |
| 1117 | internal_fatal(!datafile->fileno, "DBENGINE: datafile supplied does not have a number"); |
| 1118 | |
| 1119 | bool added = true; |
| 1120 | PGC_PAGE *page = pgc_page_add_and_acquire(open_cache, page_entry, &added); |
| 1121 | int tries = 100; |
| 1122 | while(!added && page_entry.end_time_s > pgc_page_end_time_s(page) && tries--) { |
| 1123 | pgc_page_to_clean_evict_or_release(open_cache, page); |
| 1124 | page = pgc_page_add_and_acquire(open_cache, page_entry, &added); |
| 1125 | } |
| 1126 | |
| 1127 | if(!added) { |
| 1128 | datafile_release(datafile, DATAFILE_ACQUIRE_OPEN_CACHE); |
| 1129 | |
| 1130 | internal_fatal(page_entry.end_time_s > pgc_page_end_time_s(page), |
| 1131 | "DBENGINE: cannot add longer page to open cache"); |
| 1132 | } |
| 1133 | |
| 1134 | pgc_page_release(open_cache, (PGC_PAGE *)page); |
| 1135 | } |
| 1136 | |
| 1137 | int64_t dynamic_open_cache_size(void) { |
| 1138 | int64_t main_wanted_cache_size = pgc_get_wanted_cache_size(main_cache); |
| 1139 | int64_t target_size = main_wanted_cache_size / 100 * 5; |
| 1140 | |
| 1141 | if(target_size < 2 * 1024 * 1024) |
| 1142 | target_size = 2 * 1024 * 1024; |
| 1143 | |
| 1144 | int64_t main_current_cache_size = pgc_get_current_cache_size(main_cache); |
| 1145 | |
| 1146 | int64_t main_free_cache_size = (main_wanted_cache_size > main_current_cache_size) ? |
| 1147 | main_wanted_cache_size - main_current_cache_size : 0; |
| 1148 | |
| 1149 | return target_size + main_free_cache_size; |
| 1150 | } |
| 1151 | |
| 1152 | int64_t dynamic_extent_cache_size(void) { |
| 1153 | int64_t main_wanted_cache_size = pgc_get_wanted_cache_size(main_cache); |
| 1154 | int64_t target_size = main_wanted_cache_size / 100 * 30; |
| 1155 | |
| 1156 | if(target_size < 5 * 1024 * 1024) |
| 1157 | target_size = 5 * 1024 * 1024; |
| 1158 | |
| 1159 | int64_t main_current_cache_size = pgc_get_current_cache_size(main_cache); |
| 1160 | |
| 1161 | int64_t main_free_cache_size = (main_wanted_cache_size > main_current_cache_size) ? |
| 1162 | main_wanted_cache_size - main_current_cache_size : 0; |
| 1163 | |
| 1164 | return target_size + main_free_cache_size; |
| 1165 | } |
| 1166 | |
| 1167 | size_t pgc_main_nominal_page_size(void *data) { |
| 1168 | return pgd_buffer_memory_footprint(data); |
| 1169 | } |
| 1170 | |
| 1171 | void pgc_and_mrg_initialize(void) |
| 1172 | { |
| 1173 | main_mrg = mrg_create(); |
| 1174 | |
| 1175 | size_t target_cache_size = (size_t)default_rrdeng_page_cache_mb * 1024ULL * 1024ULL; |
| 1176 | size_t main_cache_size = (target_cache_size / 100) * 70; |
| 1177 | size_t open_cache_size = 0; |
| 1178 | size_t extent_cache_size = (target_cache_size / 100) * 30; |
| 1179 | |
| 1180 | if(extent_cache_size < 5 * 1024 * 1024) { |
| 1181 | extent_cache_size = 5 * 1024 * 1024; |
| 1182 | main_cache_size = target_cache_size - extent_cache_size; |
| 1183 | } |
| 1184 | |
| 1185 | extent_cache_size += (size_t)(default_rrdeng_extent_cache_mb * 1024ULL * 1024ULL); |
| 1186 | |
| 1187 | main_cache = pgc_create( |
| 1188 | "MAIN_PGC", |
| 1189 | main_cache_size, |
| 1190 | main_cache_free_clean_page_callback, |
| 1191 | (size_t) rrdeng_pages_per_extent, |
| 1192 | main_cache_flush_dirty_page_init_callback, |
| 1193 | main_cache_flush_dirty_page_callback, |
| 1194 | 2, |
| 1195 | pgc_max_evictors(), |
| 1196 | 1000, |
| 1197 | 1, |
| 1198 | PGC_OPTIONS_AUTOSCALE | PGC_OPTIONS_EVICT_PAGES_NO_INLINE, |
| 1199 | 0, |
| 1200 | 0 |
| 1201 | ); |
| 1202 | pgc_set_nominal_page_size_callback(main_cache, pgc_main_nominal_page_size); |
| 1203 | |
| 1204 | open_cache = pgc_create( |
| 1205 | "OPEN_PGC", |
| 1206 | open_cache_size, |
| 1207 | open_cache_free_clean_page_callback, |
| 1208 | 2, |
| 1209 | NULL, |
| 1210 | open_cache_flush_dirty_page_callback, |
| 1211 | 1, |
| 1212 | pgc_max_evictors(), |
| 1213 | 1000, |
| 1214 | 1, |
| 1215 | PGC_OPTIONS_AUTOSCALE | PGC_OPTIONS_FLUSH_PAGES_NO_INLINE | PGC_OPTIONS_EVICT_PAGES_NO_INLINE, |
| 1216 | 0, |
| 1217 | sizeof(struct extent_io_data) |
| 1218 | ); |
| 1219 | pgc_set_dynamic_target_cache_size_callback(open_cache, dynamic_open_cache_size); |
| 1220 | |
| 1221 | extent_cache = pgc_create( |
| 1222 | "EXTENT_PGC", |
| 1223 | extent_cache_size, |
| 1224 | extent_cache_free_clean_page_callback, |
| 1225 | 2, |
| 1226 | NULL, |
| 1227 | extent_cache_flush_dirty_page_callback, |
| 1228 | 1, |
| 1229 | pgc_max_evictors(), |
| 1230 | 1000, |
| 1231 | 1, |
| 1232 | PGC_OPTIONS_AUTOSCALE | PGC_OPTIONS_FLUSH_PAGES_NO_INLINE | PGC_OPTIONS_EVICT_PAGES_NO_INLINE, // no flushing needed |
| 1233 | 0, |
| 1234 | 0 |
| 1235 | ); |
| 1236 | pgc_set_dynamic_target_cache_size_callback(extent_cache, dynamic_extent_cache_size); |
| 1237 | } |