| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "page.h" |
| 4 | |
| 5 | #include "libnetdata/libnetdata.h" |
| 6 | |
| 7 | typedef enum __attribute__((packed)) { |
| 8 | PAGE_OPTION_ALL_VALUES_EMPTY = (1 << 0), |
| 9 | PAGE_OPTION_ARAL_MARKED = (1 << 1), |
| 10 | PAGE_OPTION_ARAL_UNMARKED = (1 << 2), |
| 11 | } PAGE_OPTIONS; |
| 12 | |
| 13 | typedef enum __attribute__((packed)) { |
| 14 | PGD_STATE_CREATED_FROM_COLLECTOR = (1 << 0), |
| 15 | PGD_STATE_CREATED_FROM_DISK = (1 << 1), |
| 16 | PGD_STATE_SCHEDULED_FOR_FLUSHING = (1 << 2), |
| 17 | PGD_STATE_FLUSHED_TO_DISK = (1 << 3), |
| 18 | } PGD_STATES; |
| 19 | |
| 20 | typedef struct { |
| 21 | uint8_t *data; |
| 22 | uint16_t size; |
| 23 | } page_raw_t; |
| 24 | |
| 25 | typedef struct { |
| 26 | gorilla_writer_t *writer; |
| 27 | uint16_t num_buffers; |
| 28 | } page_gorilla_t; |
| 29 | |
| 30 | struct pgd { |
| 31 | // the used number of slots in the page |
| 32 | uint16_t used; |
| 33 | |
| 34 | // the total number of slots available in the page |
| 35 | uint16_t slots; |
| 36 | |
| 37 | // the page type |
| 38 | uint8_t type; |
| 39 | |
| 40 | // the partition this pgd was allocated from |
| 41 | uint8_t partition; |
| 42 | |
| 43 | // options related to the page |
| 44 | PAGE_OPTIONS options; |
| 45 | |
| 46 | PGD_STATES states; |
| 47 | |
| 48 | union { |
| 49 | page_raw_t raw; |
| 50 | page_gorilla_t gorilla; |
| 51 | }; |
| 52 | }; |
| 53 | |
| 54 | static PRINTFLIKE(2, 3) void pgd_fatal(const PGD *pg, const char *fmt, ...) { |
| 55 | BUFFER *wb = buffer_create(0, NULL); |
| 56 | |
| 57 | va_list args; |
| 58 | va_start(args, fmt); |
| 59 | buffer_vsprintf(wb, fmt, args); |
| 60 | va_end(args); |
| 61 | |
| 62 | buffer_strcat(wb, " - pgd: { "); |
| 63 | |
| 64 | { |
| 65 | buffer_strcat(wb, "type: "); |
| 66 | bool added = false; |
| 67 | |
| 68 | if (pg->type == RRDENG_PAGE_TYPE_ARRAY_32BIT) { |
| 69 | buffer_sprintf(wb, "%s", "ARRAY_32BIT"); |
| 70 | added = true; |
| 71 | } |
| 72 | |
| 73 | if (pg->type == RRDENG_PAGE_TYPE_ARRAY_TIER1) { |
| 74 | buffer_sprintf(wb, added ? "|%s" : "%s", "ARRAY_TIER1"); |
| 75 | added = true; |
| 76 | } |
| 77 | |
| 78 | if (pg->type == RRDENG_PAGE_TYPE_GORILLA_32BIT) { |
| 79 | buffer_sprintf(wb, added ? "|%s" : "%s", "GORILLA_32BIT"); |
| 80 | added = true; |
| 81 | } |
| 82 | |
| 83 | if (!added) { |
| 84 | int type = pg->type; |
| 85 | buffer_sprintf(wb, "%d", type); |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | { |
| 90 | int used = pg->used; |
| 91 | int slots = pg->slots; |
| 92 | int partition = pg->partition; |
| 93 | buffer_sprintf(wb, ", used: %d, slots: %d, partition: %d", used, slots, partition); |
| 94 | } |
| 95 | |
| 96 | { |
| 97 | buffer_strcat(wb, ", state: "); |
| 98 | bool added = false; |
| 99 | |
| 100 | if (pg->states == PGD_STATE_CREATED_FROM_COLLECTOR) { |
| 101 | buffer_sprintf(wb, "%s", "CREATED_FROM_COLLECTOR"); |
| 102 | added = true; |
| 103 | } |
| 104 | |
| 105 | if (pg->states == PGD_STATE_CREATED_FROM_DISK) { |
| 106 | buffer_sprintf(wb, added ? "|%s" : "%s", "CREATED_FROM_DISK"); |
| 107 | added = true; |
| 108 | } |
| 109 | |
| 110 | if (pg->states == PGD_STATE_SCHEDULED_FOR_FLUSHING) { |
| 111 | buffer_sprintf(wb, added ? "|%s" : "%s", "SCHEDULED_FOR_FLUSHING"); |
| 112 | added = true; |
| 113 | } |
| 114 | |
| 115 | if (pg->states == PGD_STATE_FLUSHED_TO_DISK) { |
| 116 | buffer_sprintf(wb, added ? "|%s" : "%s", "FLUSHED_TO_DISK"); |
| 117 | added = true; |
| 118 | } |
| 119 | |
| 120 | if (!added) { |
| 121 | int state = pg->states; |
| 122 | buffer_sprintf(wb, "%d", state); |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | { |
| 127 | buffer_strcat(wb, ", options: "); |
| 128 | bool added = false; |
| 129 | |
| 130 | if (pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) { |
| 131 | buffer_sprintf(wb, "%s", "ALL_VALUES_EMPTY"); |
| 132 | added = true; |
| 133 | } |
| 134 | |
| 135 | if (pg->options & PAGE_OPTION_ARAL_MARKED) { |
| 136 | buffer_sprintf(wb, added ? "|%s" : "%s", "ARAL_MARKED"); |
| 137 | added = true; |
| 138 | } |
| 139 | |
| 140 | if (pg->options & PAGE_OPTION_ARAL_UNMARKED) { |
| 141 | buffer_sprintf(wb, added ? "|%s" : "%s", "ARAL_UNMARKED"); |
| 142 | added = true; |
| 143 | } |
| 144 | |
| 145 | if (!added) { |
| 146 | int options = pg->options; |
| 147 | buffer_sprintf(wb, "%d", options); |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | buffer_strcat(wb, " }"); |
| 152 | |
| 153 | fatal("%s", buffer_tostring(wb)); |
| 154 | } |
| 155 | |
| 156 | // ---------------------------------------------------------------------------- |
| 157 | // memory management |
| 158 | |
| 159 | // deduplicate aral sizes, if the delta is below this number of bytes |
| 160 | #define ARAL_TOLERANCE_TO_DEDUP 7 |
| 161 | |
| 162 | // max, we use as many as the cpu cores |
| 163 | // cannot be bigger than 256, due to struct pgd->partition (uint8_t) |
| 164 | #define PGD_ARAL_PARTITIONS_MAX 256 |
| 165 | |
| 166 | struct { |
| 167 | int64_t padding_used; |
| 168 | size_t partitions; |
| 169 | |
| 170 | size_t sizeof_pgd; |
| 171 | size_t sizeof_gorilla_writer_t; |
| 172 | size_t sizeof_gorilla_buffer_32bit; |
| 173 | |
| 174 | ARAL *aral_pgd[PGD_ARAL_PARTITIONS_MAX]; |
| 175 | ARAL *aral_gorilla_buffer[PGD_ARAL_PARTITIONS_MAX]; |
| 176 | ARAL *aral_gorilla_writer[PGD_ARAL_PARTITIONS_MAX]; |
| 177 | } pgd_alloc_globals = { 0 }; |
| 178 | |
| 179 | #if RRD_STORAGE_TIERS != 5 |
| 180 | #error "You need to update the slots reserved for storage tiers" |
| 181 | #endif |
| 182 | |
| 183 | static struct aral_statistics pgd_aral_statistics = { 0 }; |
| 184 | |
| 185 | static size_t aral_sizes_delta; |
| 186 | static size_t aral_sizes_count; |
| 187 | static size_t aral_sizes[] = { |
| 188 | // // leave space for the storage tier page sizes |
| 189 | [RRD_STORAGE_TIERS - 5] = 0, |
| 190 | [RRD_STORAGE_TIERS - 4] = 0, |
| 191 | [RRD_STORAGE_TIERS - 3] = 0, |
| 192 | [RRD_STORAGE_TIERS - 2] = 0, |
| 193 | [RRD_STORAGE_TIERS - 1] = 0, |
| 194 | |
| 195 | // gorilla buffer sizes |
| 196 | RRDENG_GORILLA_32BIT_BUFFER_SIZE, |
| 197 | RRDENG_GORILLA_32BIT_BUFFER_SIZE * 2, |
| 198 | RRDENG_GORILLA_32BIT_BUFFER_SIZE * 3, |
| 199 | RRDENG_GORILLA_32BIT_BUFFER_SIZE * 4, |
| 200 | |
| 201 | // our structures |
| 202 | sizeof(gorilla_writer_t), |
| 203 | sizeof(PGD), |
| 204 | |
| 205 | // per 512B |
| 206 | 512, 1024, 1536, 2048, 5 * 512, 6 * 512, 7 * 512, 8 * 512, /* 9 * 512, */ |
| 207 | |
| 208 | // per 1KiB |
| 209 | // 5 * 1024, 6 * 1024, 7 * 1024, 8 * 1024, 9 * 1024, 10 * 1024, 11 * 1024, |
| 210 | // 12 * 1024, 13 * 1024, 14 * 1024, 15 * 1024, 16 * 1024, 17 * 1024, 18 * 1024, |
| 211 | // 19 * 1024, 20 * 1024, 21 * 1024, 22 * 1024, 23 * 1024, 24 * 1024, 25 * 1024, |
| 212 | // 26 * 1024, 27 * 1024, 28 * 1024, 29 * 1024, 30 * 1024, 31 * 1024, 32 * 1024, |
| 213 | |
| 214 | // test to see if 4KiB has less overheads than 1KiB |
| 215 | 8 * 1024, 12 * 1024, 16 * 1024, 20 * 1024, 24 * 1024, 28 * 1024, 32 * 1024, |
| 216 | |
| 217 | // per 4KiB |
| 218 | 36 * 1024, 40 * 1024, 44 * 1024, 48 * 1024, 52 * 1024, 56 * 1024, 60 * 1024, |
| 219 | 64 * 1024, 68 * 1024, 72 * 1024, 76 * 1024, 80 * 1024, 84 * 1024, 88 * 1024, |
| 220 | 92 * 1024, 96 * 1024, 100 * 1024, 104 * 1024, 108 * 1024, 112 * 1024, 116 * 1024, |
| 221 | 120 * 1024, 124 * 1024, 128 * 1024, |
| 222 | }; |
| 223 | static ARAL **arals = NULL; |
| 224 | |
| 225 | #define arals_slot(slot, partition) ((partition) * aral_sizes_count + (slot)) |
| 226 | static ARAL *pgd_get_aral_by_size_and_partition(size_t size, size_t partition); |
| 227 | |
| 228 | size_t pgd_padding_bytes(void) { |
| 229 | int64_t x = __atomic_load_n(&pgd_alloc_globals.padding_used, __ATOMIC_RELAXED); |
| 230 | return (x > 0) ? x : 0; |
| 231 | } |
| 232 | |
| 233 | struct aral_statistics *pgd_aral_stats(void) { |
| 234 | return &pgd_aral_statistics; |
| 235 | } |
| 236 | |
| 237 | int aral_size_sort_compare(const void *a, const void *b) { |
| 238 | size_t size_a = *(const size_t *)a; |
| 239 | size_t size_b = *(const size_t *)b; |
| 240 | return (size_a > size_b) - (size_a < size_b); |
| 241 | } |
| 242 | |
| 243 | void pgd_init_arals(void) { |
| 244 | size_t partitions = netdata_conf_cpus(); |
| 245 | if(partitions < 4) partitions = 4; |
| 246 | if(partitions > PGD_ARAL_PARTITIONS_MAX) partitions = PGD_ARAL_PARTITIONS_MAX; |
| 247 | pgd_alloc_globals.partitions = partitions; |
| 248 | |
| 249 | aral_sizes_count = _countof(aral_sizes); |
| 250 | |
| 251 | for(size_t i = 0; i < RRD_STORAGE_TIERS ;i++) |
| 252 | aral_sizes[i] = tier_page_size[i]; |
| 253 | |
| 254 | if(!netdata_conf_is_parent()) { |
| 255 | // this agent is not a parent |
| 256 | // do not use ARAL for sizes above 4KiB |
| 257 | for(size_t i = RRD_STORAGE_TIERS ; i < _countof(aral_sizes) ;i++) { |
| 258 | if(aral_sizes[i] > 4096) |
| 259 | aral_sizes[i] = 0; |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | size_t max_delta = 0; |
| 264 | for(size_t i = 0; i < aral_sizes_count ;i++) { |
| 265 | size_t wanted = aral_sizes[i]; |
| 266 | size_t usable = aral_sizes[i]; /* aral_allocation_slot_size(wanted, true);*/ |
| 267 | internal_fatal(usable < wanted, "usable cannot be less than wanted"); |
| 268 | if(usable > wanted && usable - wanted > max_delta) |
| 269 | max_delta = usable - wanted; |
| 270 | |
| 271 | aral_sizes[i] = usable; |
| 272 | } |
| 273 | aral_sizes_delta = max_delta + ARAL_TOLERANCE_TO_DEDUP; |
| 274 | |
| 275 | // sort the array |
| 276 | qsort(aral_sizes, aral_sizes_count, sizeof(size_t), aral_size_sort_compare); |
| 277 | |
| 278 | // deduplicate (with some tolerance) |
| 279 | size_t unique_count = 1; |
| 280 | for (size_t i = 1; i < aral_sizes_count; ++i) { |
| 281 | if (aral_sizes[i] > aral_sizes[unique_count - 1] + aral_sizes_delta) |
| 282 | aral_sizes[unique_count++] = aral_sizes[i]; |
| 283 | else |
| 284 | aral_sizes[unique_count - 1] = aral_sizes[i]; |
| 285 | } |
| 286 | aral_sizes_count = unique_count; |
| 287 | |
| 288 | // clear the rest |
| 289 | for(size_t i = unique_count; i < _countof(aral_sizes) ;i++) |
| 290 | aral_sizes[i] = 0; |
| 291 | |
| 292 | // allocate all the arals |
| 293 | arals = callocz(aral_sizes_count * pgd_alloc_globals.partitions, sizeof(ARAL *)); |
| 294 | for(size_t slot = 0; slot < aral_sizes_count ; slot++) { |
| 295 | for(size_t partition = 0; partition < pgd_alloc_globals.partitions; partition++) { |
| 296 | |
| 297 | if(partition > 0 && aral_sizes[slot] > 128) { |
| 298 | // do not create partitions for sizes above 128 bytes |
| 299 | // use the first partition for all of them |
| 300 | arals[arals_slot(slot, partition)] = arals[arals_slot(slot, 0)]; |
| 301 | continue; |
| 302 | } |
| 303 | |
| 304 | char buf[32]; |
| 305 | snprintfz(buf, sizeof(buf), "pgd-%zu-%zu", aral_sizes[slot], partition); |
| 306 | |
| 307 | arals[arals_slot(slot, partition)] = aral_create( |
| 308 | buf, |
| 309 | aral_sizes[slot], |
| 310 | 0, |
| 311 | 0, |
| 312 | &pgd_aral_statistics, |
| 313 | NULL, NULL, false, false, true); |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | for(size_t p = 0; p < pgd_alloc_globals.partitions ;p++) { |
| 318 | pgd_alloc_globals.aral_pgd[p] = pgd_get_aral_by_size_and_partition(sizeof(PGD), p); |
| 319 | pgd_alloc_globals.aral_gorilla_writer[p] = pgd_get_aral_by_size_and_partition(sizeof(gorilla_writer_t), p); |
| 320 | pgd_alloc_globals.aral_gorilla_buffer[p] = pgd_get_aral_by_size_and_partition(RRDENG_GORILLA_32BIT_BUFFER_SIZE, p); |
| 321 | |
| 322 | internal_fatal(!pgd_alloc_globals.aral_pgd[p] || |
| 323 | !pgd_alloc_globals.aral_gorilla_writer[p] || |
| 324 | !pgd_alloc_globals.aral_gorilla_buffer[p] |
| 325 | , "required PGD aral sizes not found"); |
| 326 | } |
| 327 | |
| 328 | pgd_alloc_globals.sizeof_pgd = aral_actual_element_size(pgd_alloc_globals.aral_pgd[0]); |
| 329 | pgd_alloc_globals.sizeof_gorilla_writer_t = aral_actual_element_size(pgd_alloc_globals.aral_gorilla_writer[0]); |
| 330 | pgd_alloc_globals.sizeof_gorilla_buffer_32bit = aral_actual_element_size(pgd_alloc_globals.aral_gorilla_buffer[0]); |
| 331 | |
| 332 | pulse_aral_register_statistics(&pgd_aral_statistics, "pgd"); |
| 333 | } |
| 334 | |
| 335 | static ARAL *pgd_get_aral_by_size_and_partition(size_t size, size_t partition) { |
| 336 | internal_fatal(partition >= pgd_alloc_globals.partitions, "Wrong partition %zu", partition); |
| 337 | |
| 338 | size_t slot; |
| 339 | |
| 340 | if (size <= aral_sizes[0]) |
| 341 | slot = 0; |
| 342 | |
| 343 | else if (size > aral_sizes[aral_sizes_count - 1]) |
| 344 | return NULL; |
| 345 | |
| 346 | else { |
| 347 | // binary search for the smallest size >= requested size |
| 348 | size_t low = 0, high = aral_sizes_count - 1; |
| 349 | while (low < high) { |
| 350 | size_t mid = low + (high - low) / 2; |
| 351 | if (aral_sizes[mid] >= size) |
| 352 | high = mid; |
| 353 | else |
| 354 | low = mid + 1; |
| 355 | } |
| 356 | slot = low; // This is the smallest index where aral_sizes[slot] >= size |
| 357 | } |
| 358 | internal_fatal(slot >= aral_sizes_count || aral_sizes[slot] < size, "Invalid PGD size binary search"); |
| 359 | |
| 360 | ARAL *ar = arals[arals_slot(slot, partition)]; |
| 361 | internal_fatal(!ar || aral_requested_element_size(ar) < size, "Invalid PGD aral lookup"); |
| 362 | return ar; |
| 363 | } |
| 364 | |
| 365 | static ALWAYS_INLINE gorilla_writer_t *pgd_gorilla_writer_alloc(size_t partition) { |
| 366 | internal_fatal(partition >= pgd_alloc_globals.partitions, "invalid gorilla writer partition %zu", partition); |
| 367 | return aral_mallocz_marked(pgd_alloc_globals.aral_gorilla_writer[partition]); |
| 368 | } |
| 369 | |
| 370 | static ALWAYS_INLINE gorilla_buffer_t *pgd_gorilla_buffer_alloc(size_t partition) { |
| 371 | internal_fatal(partition >= pgd_alloc_globals.partitions, "invalid gorilla buffer partition %zu", partition); |
| 372 | return aral_mallocz_marked(pgd_alloc_globals.aral_gorilla_buffer[partition]); |
| 373 | } |
| 374 | |
| 375 | static ALWAYS_INLINE PGD *pgd_alloc(bool for_collector) { |
| 376 | size_t partition = gettid_cached() % pgd_alloc_globals.partitions; |
| 377 | PGD *pgd; |
| 378 | |
| 379 | if(for_collector) |
| 380 | pgd = aral_mallocz_marked(pgd_alloc_globals.aral_pgd[partition]); |
| 381 | else |
| 382 | pgd = aral_mallocz(pgd_alloc_globals.aral_pgd[partition]); |
| 383 | |
| 384 | pgd->partition = partition; |
| 385 | return pgd; |
| 386 | } |
| 387 | |
| 388 | static ALWAYS_INLINE void *pgd_data_alloc(size_t size, size_t partition, bool for_collector) { |
| 389 | ARAL *ar = pgd_get_aral_by_size_and_partition(size, partition); |
| 390 | if(ar) { |
| 391 | int64_t padding = (int64_t)aral_requested_element_size(ar) - (int64_t)size; |
| 392 | __atomic_add_fetch(&pgd_alloc_globals.padding_used, padding, __ATOMIC_RELAXED); |
| 393 | |
| 394 | if(for_collector) |
| 395 | return aral_mallocz_marked(ar); |
| 396 | else |
| 397 | return aral_mallocz(ar); |
| 398 | } |
| 399 | else |
| 400 | return mallocz(size); |
| 401 | } |
| 402 | |
| 403 | static ALWAYS_INLINE void pgd_data_free(void *page, size_t size, size_t partition) { |
| 404 | ARAL *ar = pgd_get_aral_by_size_and_partition(size, partition); |
| 405 | if(ar) { |
| 406 | int64_t padding = (int64_t)aral_requested_element_size(ar) - (int64_t)size; |
| 407 | __atomic_sub_fetch(&pgd_alloc_globals.padding_used, padding, __ATOMIC_RELAXED); |
| 408 | |
| 409 | aral_freez(ar, page); |
| 410 | } |
| 411 | else |
| 412 | freez(page); |
| 413 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_TIER1_ARAL); |
| 414 | } |
| 415 | |
| 416 | static ALWAYS_INLINE void pgd_data_unmark(void *page, size_t size, size_t partition) { |
| 417 | if(!page) return; |
| 418 | |
| 419 | ARAL *ar = pgd_get_aral_by_size_and_partition(size, partition); |
| 420 | if(ar) |
| 421 | aral_unmark_allocation(ar, page); |
| 422 | } |
| 423 | |
| 424 | static size_t pgd_data_footprint(size_t size, size_t partition) { |
| 425 | ARAL *ar = pgd_get_aral_by_size_and_partition(size, partition); |
| 426 | if(ar) |
| 427 | return aral_actual_element_size(ar); |
| 428 | else |
| 429 | return size; |
| 430 | } |
| 431 | |
| 432 | // ---------------------------------------------------------------------------- |
| 433 | |
| 434 | ALWAYS_INLINE void *dbengine_extent_alloc(size_t size) { |
| 435 | return pgd_data_alloc(size, 0, false); |
| 436 | } |
| 437 | |
| 438 | ALWAYS_INLINE void dbengine_extent_free(void *extent, size_t size) { |
| 439 | pgd_data_free(extent, size, 0); |
| 440 | } |
| 441 | |
| 442 | // ---------------------------------------------------------------------------- |
| 443 | // management api |
| 444 | |
| 445 | ALWAYS_INLINE PGD *pgd_create(uint8_t type, uint32_t slots) { |
| 446 | |
| 447 | PGD *pg = pgd_alloc(true); // this is malloc'd ! |
| 448 | pg->type = type; |
| 449 | pg->states = PGD_STATE_CREATED_FROM_COLLECTOR; |
| 450 | pg->options = PAGE_OPTION_ALL_VALUES_EMPTY | PAGE_OPTION_ARAL_MARKED; |
| 451 | |
| 452 | pg->used = 0; |
| 453 | pg->slots = slots; |
| 454 | |
| 455 | switch (type) { |
| 456 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 457 | internal_fatal(slots == 1, |
| 458 | "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type); |
| 459 | |
| 460 | // allocate new gorilla writer |
| 461 | pg->gorilla.writer = pgd_gorilla_writer_alloc(pg->partition); |
| 462 | |
| 463 | // allocate new gorilla buffer |
| 464 | gorilla_buffer_t *gbuf = pgd_gorilla_buffer_alloc(pg->partition); |
| 465 | memset(gbuf, 0, RRDENG_GORILLA_32BIT_BUFFER_SIZE); |
| 466 | pulse_gorilla_hot_buffer_added(); |
| 467 | |
| 468 | *pg->gorilla.writer = gorilla_writer_init(gbuf, RRDENG_GORILLA_32BIT_BUFFER_SLOTS); |
| 469 | pg->gorilla.num_buffers = 1; |
| 470 | |
| 471 | break; |
| 472 | } |
| 473 | |
| 474 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 475 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: { |
| 476 | uint32_t size = slots * page_type_size[type]; |
| 477 | |
| 478 | internal_fatal(!size || slots == 1, |
| 479 | "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type); |
| 480 | |
| 481 | pg->raw.size = size; |
| 482 | pg->raw.data = pgd_data_alloc(size, pg->partition, true); |
| 483 | break; |
| 484 | } |
| 485 | |
| 486 | default: |
| 487 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, type); |
| 488 | aral_freez(pgd_alloc_globals.aral_pgd[pg->partition], pg); |
| 489 | pg = PGD_EMPTY; |
| 490 | break; |
| 491 | } |
| 492 | |
| 493 | return pg; |
| 494 | } |
| 495 | |
| 496 | ALWAYS_INLINE PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size) { |
| 497 | |
| 498 | if (!size || size < page_type_size[type]) |
| 499 | return PGD_EMPTY; |
| 500 | |
| 501 | PGD *pg = pgd_alloc(false); // this is malloc'd ! |
| 502 | pg->type = type; |
| 503 | pg->states = PGD_STATE_CREATED_FROM_DISK; |
| 504 | pg->options = PAGE_OPTION_ARAL_UNMARKED; |
| 505 | |
| 506 | switch (type) |
| 507 | { |
| 508 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: |
| 509 | internal_fatal(size == 0, "Asked to create page with 0 data!!!"); |
| 510 | internal_fatal(size % sizeof(uint32_t), "Unaligned gorilla buffer size"); |
| 511 | internal_fatal(size % RRDENG_GORILLA_32BIT_BUFFER_SIZE, "Expected size to be a multiple of %zu-bytes", |
| 512 | RRDENG_GORILLA_32BIT_BUFFER_SIZE); |
| 513 | |
| 514 | pg->raw.data = (void *)pgd_data_alloc(size, pg->partition, false); |
| 515 | pg->raw.size = size; |
| 516 | |
| 517 | memcpy(pg->raw.data, base, pg->raw.size); |
| 518 | |
| 519 | uint32_t total_entries = gorilla_buffer_patch((void *) pg->raw.data); |
| 520 | pg->used = total_entries; |
| 521 | pg->slots = pg->used; |
| 522 | break; |
| 523 | |
| 524 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 525 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: |
| 526 | pg->used = size / page_type_size[type]; |
| 527 | pg->slots = pg->used; |
| 528 | |
| 529 | pg->raw.size = size; |
| 530 | pg->raw.data = pgd_data_alloc(size, pg->partition, false); |
| 531 | memcpy(pg->raw.data, base, size); |
| 532 | break; |
| 533 | |
| 534 | default: |
| 535 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, type); |
| 536 | aral_freez(pgd_alloc_globals.aral_pgd[pg->partition], pg); |
| 537 | pg = PGD_EMPTY; |
| 538 | break; |
| 539 | } |
| 540 | |
| 541 | return pg; |
| 542 | } |
| 543 | |
| 544 | void pgd_free(PGD *pg) { |
| 545 | if (!pg || pg == PGD_EMPTY) |
| 546 | return; |
| 547 | |
| 548 | internal_fatal(pg->partition >= pgd_alloc_globals.partitions, |
| 549 | "PGD partition is invalid %u", pg->partition); |
| 550 | |
| 551 | switch (pg->type) |
| 552 | { |
| 553 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 554 | if (pg->states & PGD_STATE_CREATED_FROM_DISK) |
| 555 | { |
| 556 | internal_fatal(pg->raw.data == NULL, "Tried to free gorilla PGD loaded from disk with NULL data"); |
| 557 | |
| 558 | pgd_data_free(pg->raw.data, pg->raw.size, pg->partition); |
| 559 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_ARAL); |
| 560 | |
| 561 | pg->raw.data = NULL; |
| 562 | pg->raw.size = 0; |
| 563 | } |
| 564 | else if ((pg->states & PGD_STATE_CREATED_FROM_COLLECTOR) || |
| 565 | (pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) || |
| 566 | (pg->states & PGD_STATE_FLUSHED_TO_DISK)) |
| 567 | { |
| 568 | internal_fatal(pg->gorilla.writer == NULL, |
| 569 | "PGD does not have an active gorilla writer"); |
| 570 | |
| 571 | internal_fatal(pg->gorilla.num_buffers == 0, |
| 572 | "PGD does not have any gorilla buffers allocated"); |
| 573 | |
| 574 | while (true) { |
| 575 | gorilla_buffer_t *gbuf = gorilla_writer_drop_head_buffer(pg->gorilla.writer); |
| 576 | if (!gbuf) |
| 577 | break; |
| 578 | aral_freez(pgd_alloc_globals.aral_gorilla_buffer[pg->partition], gbuf); |
| 579 | pg->gorilla.num_buffers -= 1; |
| 580 | } |
| 581 | |
| 582 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_GLIVE); |
| 583 | |
| 584 | internal_fatal(pg->gorilla.num_buffers != 0, |
| 585 | "Could not free all gorilla writer buffers"); |
| 586 | |
| 587 | aral_freez(pgd_alloc_globals.aral_gorilla_writer[pg->partition], pg->gorilla.writer); |
| 588 | pg->gorilla.writer = NULL; |
| 589 | |
| 590 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_GWORKER); |
| 591 | } else { |
| 592 | fatal("pgd_free() called on gorilla page with unsupported state"); |
| 593 | // TODO: should we support any other states? |
| 594 | // if (!(pg->states & PGD_STATE_FLUSHED_TO_DISK)) |
| 595 | // fatal("pgd_free() is not supported yet for pages flushed to disk"); |
| 596 | } |
| 597 | |
| 598 | break; |
| 599 | } |
| 600 | |
| 601 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 602 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: |
| 603 | pgd_data_free(pg->raw.data, pg->raw.size, pg->partition); |
| 604 | break; |
| 605 | |
| 606 | default: |
| 607 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 608 | break; |
| 609 | } |
| 610 | |
| 611 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_DATA); |
| 612 | |
| 613 | aral_freez(pgd_alloc_globals.aral_pgd[pg->partition], pg); |
| 614 | |
| 615 | timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_ARAL); |
| 616 | } |
| 617 | |
| 618 | static void pgd_aral_unmark(PGD *pg) { |
| 619 | if (!pg || |
| 620 | pg == PGD_EMPTY || |
| 621 | (pg->options & PAGE_OPTION_ARAL_UNMARKED) || |
| 622 | !(pg->options & PAGE_OPTION_ARAL_MARKED)) |
| 623 | return; |
| 624 | |
| 625 | internal_fatal(pg->partition >= pgd_alloc_globals.partitions, |
| 626 | "PGD partition is invalid %u", pg->partition); |
| 627 | |
| 628 | switch (pg->type) |
| 629 | { |
| 630 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 631 | if (pg->states & PGD_STATE_CREATED_FROM_DISK) |
| 632 | pgd_data_unmark(pg->raw.data, pg->raw.size, pg->partition); |
| 633 | |
| 634 | else if ((pg->states & PGD_STATE_CREATED_FROM_COLLECTOR) || |
| 635 | (pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) || |
| 636 | (pg->states & PGD_STATE_FLUSHED_TO_DISK)) |
| 637 | { |
| 638 | internal_fatal(pg->gorilla.writer == NULL, "PGD does not have an active gorilla writer"); |
| 639 | internal_fatal(pg->gorilla.num_buffers == 0, "PGD does not have any gorilla buffers allocated"); |
| 640 | |
| 641 | gorilla_writer_aral_unmark(pg->gorilla.writer, pgd_alloc_globals.aral_gorilla_buffer[pg->partition]); |
| 642 | aral_unmark_allocation(pgd_alloc_globals.aral_gorilla_writer[pg->partition], pg->gorilla.writer); |
| 643 | } |
| 644 | else { |
| 645 | fatal("pgd_free() called on gorilla page with unsupported state"); |
| 646 | // TODO: should we support any other states? |
| 647 | // if (!(pg->states & PGD_STATE_FLUSHED_TO_DISK)) |
| 648 | // fatal("pgd_free() is not supported yet for pages flushed to disk"); |
| 649 | } |
| 650 | |
| 651 | break; |
| 652 | } |
| 653 | |
| 654 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 655 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: |
| 656 | pgd_data_unmark(pg->raw.data, pg->raw.size, pg->partition); |
| 657 | break; |
| 658 | |
| 659 | default: |
| 660 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 661 | break; |
| 662 | } |
| 663 | |
| 664 | aral_unmark_allocation(pgd_alloc_globals.aral_pgd[pg->partition], pg); |
| 665 | |
| 666 | // make sure we will not do this again |
| 667 | pg->options |= PAGE_OPTION_ARAL_UNMARKED; |
| 668 | } |
| 669 | |
| 670 | // ---------------------------------------------------------------------------- |
| 671 | // utility functions |
| 672 | |
| 673 | ALWAYS_INLINE uint32_t pgd_type(PGD *pg) |
| 674 | { |
| 675 | return pg->type; |
| 676 | } |
| 677 | |
| 678 | ALWAYS_INLINE bool pgd_is_empty(PGD *pg) |
| 679 | { |
| 680 | if (!pg) |
| 681 | return true; |
| 682 | |
| 683 | if (pg == PGD_EMPTY) |
| 684 | return true; |
| 685 | |
| 686 | if (pg->used == 0) |
| 687 | return true; |
| 688 | |
| 689 | if (pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) |
| 690 | return true; |
| 691 | |
| 692 | return false; |
| 693 | } |
| 694 | |
| 695 | ALWAYS_INLINE uint32_t pgd_slots_used(PGD *pg) |
| 696 | { |
| 697 | if (!pg) |
| 698 | return 0; |
| 699 | |
| 700 | if (pg == PGD_EMPTY) |
| 701 | return 0; |
| 702 | |
| 703 | return pg->used; |
| 704 | } |
| 705 | |
| 706 | ALWAYS_INLINE uint32_t pgd_capacity(PGD *pg) { |
| 707 | if (!pg) |
| 708 | return 0; |
| 709 | |
| 710 | if (pg == PGD_EMPTY) |
| 711 | return 0; |
| 712 | |
| 713 | return pg->slots; |
| 714 | } |
| 715 | |
| 716 | // return the overall memory footprint of the page, including all its structures and overheads |
| 717 | ALWAYS_INLINE uint32_t pgd_memory_footprint(PGD *pg) |
| 718 | { |
| 719 | if (!pg) |
| 720 | return 0; |
| 721 | |
| 722 | if (pg == PGD_EMPTY) |
| 723 | return 0; |
| 724 | |
| 725 | size_t footprint = pgd_alloc_globals.sizeof_pgd; |
| 726 | |
| 727 | switch (pg->type) { |
| 728 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 729 | if (pg->states & PGD_STATE_CREATED_FROM_DISK) |
| 730 | footprint += pgd_data_footprint(pg->raw.size, pg->partition); |
| 731 | |
| 732 | else { |
| 733 | footprint += pgd_alloc_globals.sizeof_gorilla_writer_t; |
| 734 | footprint += pg->gorilla.num_buffers * pgd_alloc_globals.sizeof_gorilla_buffer_32bit; |
| 735 | } |
| 736 | break; |
| 737 | } |
| 738 | |
| 739 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 740 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: |
| 741 | footprint += pgd_data_footprint(pg->raw.size, pg->partition); |
| 742 | break; |
| 743 | |
| 744 | default: |
| 745 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 746 | break; |
| 747 | } |
| 748 | |
| 749 | return footprint; |
| 750 | } |
| 751 | |
| 752 | // return the nominal buffer size depending on the page type - used by the PGC histogram |
| 753 | uint32_t pgd_buffer_memory_footprint(PGD *pg) |
| 754 | { |
| 755 | if (!pg) |
| 756 | return 0; |
| 757 | |
| 758 | if (pg == PGD_EMPTY) |
| 759 | return 0; |
| 760 | |
| 761 | size_t footprint = 0; |
| 762 | |
| 763 | switch (pg->type) { |
| 764 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 765 | if (pg->states & PGD_STATE_CREATED_FROM_DISK) |
| 766 | footprint = pg->raw.size; |
| 767 | |
| 768 | else |
| 769 | footprint = pg->gorilla.num_buffers * RRDENG_GORILLA_32BIT_BUFFER_SIZE; |
| 770 | break; |
| 771 | } |
| 772 | |
| 773 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 774 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: |
| 775 | footprint = pg->raw.size; |
| 776 | break; |
| 777 | |
| 778 | default: |
| 779 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 780 | break; |
| 781 | } |
| 782 | |
| 783 | return footprint; |
| 784 | } |
| 785 | |
| 786 | uint32_t pgd_disk_footprint(PGD *pg) |
| 787 | { |
| 788 | if (!pgd_slots_used(pg)) |
| 789 | return 0; |
| 790 | |
| 791 | size_t size = 0; |
| 792 | |
| 793 | // since the page is ready for flushing, let's unmark its pages to ARAL |
| 794 | pgd_aral_unmark(pg); |
| 795 | |
| 796 | switch (pg->type) { |
| 797 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 798 | if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR || |
| 799 | pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING || |
| 800 | pg->states & PGD_STATE_FLUSHED_TO_DISK) |
| 801 | { |
| 802 | internal_fatal(!pg->gorilla.writer, |
| 803 | "pgd_disk_footprint() not implemented for NULL gorilla writers"); |
| 804 | |
| 805 | internal_fatal(pg->gorilla.num_buffers == 0, |
| 806 | "Gorilla writer does not have any buffers"); |
| 807 | |
| 808 | size = pg->gorilla.num_buffers * RRDENG_GORILLA_32BIT_BUFFER_SIZE; |
| 809 | |
| 810 | if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR) |
| 811 | pulse_gorilla_tier0_page_flush( |
| 812 | gorilla_writer_actual_nbytes(pg->gorilla.writer), |
| 813 | gorilla_writer_optimal_nbytes(pg->gorilla.writer), |
| 814 | tier_page_size[0]); |
| 815 | |
| 816 | } else if (pg->states & PGD_STATE_CREATED_FROM_DISK) { |
| 817 | size = pg->raw.size; |
| 818 | } else { |
| 819 | fatal("Asked disk footprint on unknown page state"); |
| 820 | } |
| 821 | |
| 822 | break; |
| 823 | } |
| 824 | |
| 825 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 826 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: { |
| 827 | uint32_t used_size = pg->used * page_type_size[pg->type]; |
| 828 | internal_fatal(used_size > pg->raw.size, "Wrong disk footprint page size"); |
| 829 | size = used_size; |
| 830 | |
| 831 | break; |
| 832 | } |
| 833 | |
| 834 | default: |
| 835 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 836 | break; |
| 837 | } |
| 838 | |
| 839 | internal_fatal(pg->states & PGD_STATE_CREATED_FROM_DISK, |
| 840 | "Disk footprint asked for page created from disk."); |
| 841 | |
| 842 | pg->states = PGD_STATE_SCHEDULED_FOR_FLUSHING; |
| 843 | return size; |
| 844 | } |
| 845 | |
| 846 | void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size) |
| 847 | { |
| 848 | internal_fatal(pgd_disk_footprint(pg) != dst_size, "Wrong disk footprint size requested (need %u, available %u)", |
| 849 | pgd_disk_footprint(pg), dst_size); |
| 850 | |
| 851 | switch (pg->type) { |
| 852 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 853 | if ((pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) == 0) |
| 854 | fatal("Copying to extent is supported only for PGDs that are scheduled for flushing."); |
| 855 | |
| 856 | internal_fatal(!pg->gorilla.writer, |
| 857 | "pgd_copy_to_extent() not implemented for NULL gorilla writers"); |
| 858 | |
| 859 | internal_fatal(pg->gorilla.num_buffers == 0, |
| 860 | "pgd_copy_to_extent() gorilla writer does not have any buffers"); |
| 861 | |
| 862 | bool ok = gorilla_writer_serialize(pg->gorilla.writer, dst, dst_size); |
| 863 | UNUSED(ok); |
| 864 | internal_fatal(!ok, |
| 865 | "pgd_copy_to_extent() tried to serialize pg=%p, gw=%p (with dst_size=%u bytes, num_buffers=%u)", |
| 866 | pg, pg->gorilla.writer, dst_size, pg->gorilla.num_buffers); |
| 867 | break; |
| 868 | } |
| 869 | |
| 870 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 871 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: |
| 872 | memcpy(dst, pg->raw.data, dst_size); |
| 873 | break; |
| 874 | |
| 875 | default: |
| 876 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 877 | break; |
| 878 | } |
| 879 | |
| 880 | pg->states = PGD_STATE_FLUSHED_TO_DISK; |
| 881 | } |
| 882 | |
| 883 | // ---------------------------------------------------------------------------- |
| 884 | // data collection |
| 885 | |
| 886 | // returns additional memory that may have been allocated to store this point |
| 887 | ALWAYS_INLINE_HOT_FLATTEN |
| 888 | size_t pgd_append_point( |
| 889 | PGD *pg, |
| 890 | usec_t point_in_time_ut __maybe_unused, |
| 891 | NETDATA_DOUBLE n, |
| 892 | NETDATA_DOUBLE min_value, |
| 893 | NETDATA_DOUBLE max_value, |
| 894 | uint16_t count, |
| 895 | uint16_t anomaly_count, |
| 896 | SN_FLAGS flags, |
| 897 | uint32_t expected_slot) |
| 898 | { |
| 899 | if (pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) { |
| 900 | if(exit_initiated_get() == EXIT_REASON_NONE) |
| 901 | pgd_fatal(pg, "Data collection on page already scheduled for flushing"); |
| 902 | else |
| 903 | return 0; |
| 904 | } |
| 905 | |
| 906 | if (!(pg->states & PGD_STATE_CREATED_FROM_COLLECTOR)) { |
| 907 | if(exit_initiated_get() == EXIT_REASON_NONE) |
| 908 | pgd_fatal(pg, "DBENGINE: collection on page not created from a collector"); |
| 909 | else |
| 910 | return 0; |
| 911 | } |
| 912 | |
| 913 | if (unlikely(pg->used != expected_slot)) |
| 914 | pgd_fatal(pg, "DBENGINE: page is not aligned to expected slot (used %u, expected %u)", |
| 915 | pg->used, expected_slot); |
| 916 | |
| 917 | if (unlikely(pg->used >= pg->slots)) |
| 918 | pgd_fatal(pg, "DBENGINE: attempted to write beyond page size (page type %u, slots %u, used %u)", |
| 919 | pg->type, pg->slots, pg->used /* FIXME:, pg->size */); |
| 920 | |
| 921 | switch (pg->type) { |
| 922 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 923 | pg->used++; |
| 924 | storage_number t = pack_storage_number(n, flags); |
| 925 | |
| 926 | if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && does_storage_number_exist(t)) |
| 927 | pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY; |
| 928 | |
| 929 | bool ok = gorilla_writer_write(pg->gorilla.writer, t); |
| 930 | if (!ok) { |
| 931 | gorilla_buffer_t *new_buffer = pgd_gorilla_buffer_alloc(pg->partition); |
| 932 | memset(new_buffer, 0, RRDENG_GORILLA_32BIT_BUFFER_SIZE); |
| 933 | |
| 934 | gorilla_writer_add_buffer(pg->gorilla.writer, new_buffer, RRDENG_GORILLA_32BIT_BUFFER_SLOTS); |
| 935 | pg->gorilla.num_buffers += 1; |
| 936 | pulse_gorilla_hot_buffer_added(); |
| 937 | |
| 938 | ok = gorilla_writer_write(pg->gorilla.writer, t); |
| 939 | internal_fatal(ok == false, "Failed to writer value in newly allocated gorilla buffer."); |
| 940 | |
| 941 | return RRDENG_GORILLA_32BIT_BUFFER_SIZE; |
| 942 | } |
| 943 | |
| 944 | break; |
| 945 | } |
| 946 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: { |
| 947 | storage_number_tier1_t *tier12_metric_data = (storage_number_tier1_t *)pg->raw.data; |
| 948 | storage_number_tier1_t t; |
| 949 | t.sum_value = (float) n; |
| 950 | t.min_value = (float) min_value; |
| 951 | t.max_value = (float) max_value; |
| 952 | t.anomaly_count = anomaly_count; |
| 953 | t.count = count; |
| 954 | tier12_metric_data[pg->used++] = t; |
| 955 | |
| 956 | if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && fpclassify(n) != FP_NAN) |
| 957 | pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY; |
| 958 | |
| 959 | break; |
| 960 | } |
| 961 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: { |
| 962 | storage_number *tier0_metric_data = (storage_number *)pg->raw.data; |
| 963 | storage_number t = pack_storage_number(n, flags); |
| 964 | tier0_metric_data[pg->used++] = t; |
| 965 | |
| 966 | if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && does_storage_number_exist(t)) |
| 967 | pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY; |
| 968 | |
| 969 | break; |
| 970 | } |
| 971 | default: |
| 972 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 973 | break; |
| 974 | } |
| 975 | |
| 976 | return 0; |
| 977 | } |
| 978 | |
| 979 | // ---------------------------------------------------------------------------- |
| 980 | // querying with cursor |
| 981 | |
| 982 | static void pgdc_seek(PGDC *pgdc, uint32_t position) |
| 983 | { |
| 984 | PGD *pg = pgdc->pgd; |
| 985 | |
| 986 | switch (pg->type) { |
| 987 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 988 | if (pg->states & PGD_STATE_CREATED_FROM_DISK) { |
| 989 | pgdc->slots = pgdc->pgd->slots; |
| 990 | pgdc->gr = gorilla_reader_init((void *) pg->raw.data); |
| 991 | } else { |
| 992 | if (!(pg->states & PGD_STATE_CREATED_FROM_COLLECTOR) && |
| 993 | !(pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) && |
| 994 | !(pg->states & PGD_STATE_FLUSHED_TO_DISK)) |
| 995 | pgd_fatal(pg, "pgdc_seek() currently is not supported for pages created from disk."); |
| 996 | |
| 997 | if (!pg->gorilla.writer) |
| 998 | pgd_fatal(pg, "Seeking from a page without an active gorilla writer is not supported (yet)."); |
| 999 | |
| 1000 | pgdc->slots = gorilla_writer_entries(pg->gorilla.writer); |
| 1001 | pgdc->gr = gorilla_writer_get_reader(pg->gorilla.writer); |
| 1002 | } |
| 1003 | |
| 1004 | if (position > pgdc->slots) |
| 1005 | position = pgdc->slots; |
| 1006 | |
| 1007 | for (uint32_t i = 0; i != position; i++) { |
| 1008 | uint32_t value; |
| 1009 | |
| 1010 | bool ok = gorilla_reader_read(&pgdc->gr, &value); |
| 1011 | if (!ok) { |
| 1012 | // this is fine, the reader will return empty points |
| 1013 | break; |
| 1014 | } |
| 1015 | } |
| 1016 | |
| 1017 | break; |
| 1018 | } |
| 1019 | |
| 1020 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: |
| 1021 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: |
| 1022 | pgdc->slots = pgdc->pgd->used; |
| 1023 | break; |
| 1024 | |
| 1025 | default: |
| 1026 | netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type); |
| 1027 | break; |
| 1028 | } |
| 1029 | } |
| 1030 | |
| 1031 | void pgdc_reset(PGDC *pgdc, PGD *pgd, uint32_t position) |
| 1032 | { |
| 1033 | // pgd might be null and position equal to UINT32_MAX |
| 1034 | |
| 1035 | pgdc->pgd = pgd; |
| 1036 | pgdc->position = position; |
| 1037 | |
| 1038 | if (!pgd) |
| 1039 | return; |
| 1040 | |
| 1041 | if (pgd == PGD_EMPTY) |
| 1042 | return; |
| 1043 | |
| 1044 | if (position == UINT32_MAX) |
| 1045 | return; |
| 1046 | |
| 1047 | pgdc_seek(pgdc, position); |
| 1048 | } |
| 1049 | |
| 1050 | ALWAYS_INLINE_HOT_FLATTEN |
| 1051 | bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused, STORAGE_POINT *sp) |
| 1052 | { |
| 1053 | if (!pgdc->pgd || pgdc->pgd == PGD_EMPTY || pgdc->position >= pgdc->slots) |
| 1054 | { |
| 1055 | storage_point_empty(*sp, sp->start_time_s, sp->end_time_s); |
| 1056 | return false; |
| 1057 | } |
| 1058 | |
| 1059 | internal_fatal(pgdc->position != expected_position, "Wrong expected cursor position"); |
| 1060 | |
| 1061 | switch (pgdc->pgd->type) |
| 1062 | { |
| 1063 | case RRDENG_PAGE_TYPE_GORILLA_32BIT: { |
| 1064 | pgdc->position++; |
| 1065 | |
| 1066 | uint32_t n = 666666666; |
| 1067 | bool ok = gorilla_reader_read(&pgdc->gr, &n); |
| 1068 | |
| 1069 | if (ok) { |
| 1070 | sp->min = sp->max = sp->sum = unpack_storage_number(n); |
| 1071 | sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS); |
| 1072 | sp->count = 1; |
| 1073 | sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0; |
| 1074 | } else { |
| 1075 | storage_point_empty(*sp, sp->start_time_s, sp->end_time_s); |
| 1076 | } |
| 1077 | |
| 1078 | return ok; |
| 1079 | } |
| 1080 | case RRDENG_PAGE_TYPE_ARRAY_TIER1: { |
| 1081 | storage_number_tier1_t *array = (storage_number_tier1_t *) pgdc->pgd->raw.data; |
| 1082 | storage_number_tier1_t n = array[pgdc->position++]; |
| 1083 | |
| 1084 | sp->flags = n.anomaly_count ? SN_FLAG_NONE : SN_FLAG_NOT_ANOMALOUS; |
| 1085 | sp->count = n.count; |
| 1086 | sp->anomaly_count = n.anomaly_count; |
| 1087 | sp->min = n.min_value; |
| 1088 | sp->max = n.max_value; |
| 1089 | sp->sum = n.sum_value; |
| 1090 | |
| 1091 | return true; |
| 1092 | } |
| 1093 | case RRDENG_PAGE_TYPE_ARRAY_32BIT: { |
| 1094 | storage_number *array = (storage_number *) pgdc->pgd->raw.data; |
| 1095 | storage_number n = array[pgdc->position++]; |
| 1096 | |
| 1097 | sp->min = sp->max = sp->sum = unpack_storage_number(n); |
| 1098 | sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS); |
| 1099 | sp->count = 1; |
| 1100 | sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0; |
| 1101 | |
| 1102 | return true; |
| 1103 | } |
| 1104 | default: { |
| 1105 | static bool logged = false; |
| 1106 | if (!logged) |
| 1107 | { |
| 1108 | netdata_log_error("DBENGINE: unknown page type %"PRIu32" found. Cannot decode it. Ignoring its metrics.", |
| 1109 | pgd_type(pgdc->pgd)); |
| 1110 | logged = true; |
| 1111 | } |
| 1112 | |
| 1113 | storage_point_empty(*sp, sp->start_time_s, sp->end_time_s); |
| 1114 | return false; |
| 1115 | } |
| 1116 | } |
| 1117 | } |