@cryptotaxi247 / netdata-1 / commits / 62f1b458e

inline dbengine query critical path (#19537)

* inline dbengine query critical path * more dbengine inlining * optimize pgc_page_add to optimistically allocate the new entry * backfill related inlining * more inlining * simple re-organization of members of sender_state * descriptive SSL error logs * nd-poll() wanted mismatch is now a debug statement * improve SSL error handling * restore ssl error handling code * strict error checking on enable_streaming replication flag * selected functions flattening * size replication queue * fix replication step inicfg parsing; fix local variable exposure * replication sender code cleanup * process opcodes every 100ms and reorganize so that the latency per processing can be computed * more inlining at critical sections * make replication step 20m * configure replication prefetch * use the inicfg range function for libuv threads * move libuv initialize closer to the beginning * replication fixes * replication defaults tuning * aral to use 2MiB page sizes and nd_mmap() to madvise HUGEPAGE when size matches * use dedicated aral for replication sender * tune replication sender to max 6 threads * aral incoming lock now tries all partitions * health should be running together with replication * refcount added refcount_release_and_acquire_for_deletion_advanced() * minimize aral_lock() use * aral having page_lock * aral_page refcount with simple atomics * cleanup aral

Costa Tsaousis committed Feb 5, 2025 at 07:09 UTC 62f1b458e7b03cfec990bca193ef016eb25f3b0f

68 files changed +1274 -903

src/daemon/config/netdata-conf-global.c

+4 -17

@@ -75,21 +75,10 @@ void netdata_conf_glibc_malloc_initialize(size_t wanted_arenas, size_t trim_thre

75 #endif

76 }

78 -static void libuv_initialize(void) {

79 - libuv_worker_threads = (int)netdata_conf_cpus() * 6;

80 -

81 - if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS)

82 - libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;

83 -

84 - if(libuv_worker_threads > MAX_LIBUV_WORKER_THREADS)

85 - libuv_worker_threads = MAX_LIBUV_WORKER_THREADS;

86 -

87 -

88 - libuv_worker_threads = inicfg_get_number(&netdata_config, CONFIG_SECTION_GLOBAL, "libuv worker threads", libuv_worker_threads);

89 - if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS) {

90 - libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;

91 - inicfg_set_number(&netdata_config, CONFIG_SECTION_GLOBAL, "libuv worker threads", libuv_worker_threads);

92 - }

78 +void libuv_initialize(void) {

79 + libuv_worker_threads = (int)inicfg_get_number_range(

80 + &netdata_config, CONFIG_SECTION_GLOBAL, "libuv worker threads",

81 + (int)netdata_conf_cpus() * 6, MIN_LIBUV_WORKER_THREADS, MAX_LIBUV_WORKER_THREADS);

83 char buf[20 + 1];

84 snprintfz(buf, sizeof(buf) - 1, "%d", libuv_worker_threads);

@@ -120,8 +109,6 @@ void netdata_conf_section_global(void) {

109

110 os_get_system_cpus_uncached();

111 os_get_system_pid_max();

123 -

124 - libuv_initialize();

112 }

113

114 void netdata_conf_section_global_run_as_user(const char **user) {

src/daemon/config/netdata-conf-global.h

@@ -9,6 +9,7 @@ void netdata_conf_section_global(void);

9 void netdata_conf_section_global_run_as_user(const char **user);

11 size_t netdata_conf_cpus(void);

12 +void libuv_initialize(void);

14 void netdata_conf_glibc_malloc_initialize(size_t wanted_arenas, size_t trim_threshold);

src/daemon/config/netdata-conf.c

@@ -36,5 +36,6 @@ bool netdata_conf_load(char *filename, char overwrite_used, const char **user) {

37 netdata_conf_backwards_compatibility();

38 netdata_conf_section_global_run_as_user(user);

39 + libuv_initialize();

40 return ret;

41 }

src/daemon/pulse/pulse-daemon-memory.c

+1 -1

@@ -146,7 +146,7 @@ void pulse_daemon_memory_do(bool extended __maybe_unused) {

146 (collected_number)dictionary_stats_memory_total(dictionary_stats_category_functions));

147

148 rrddim_set_by_pointer(st_memory, rd_replication,

149 - (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + (collected_number)replication_sender_allocated_memory());

149 + (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + replication_sender_allocated_memory());

150 #else

151 uint64_t metadata =

152 aral_by_size_used_bytes() +

src/daemon/pulse/pulse-db-dbengine.c

+11 -11

@@ -3,7 +3,7 @@

3 #define PULSE_INTERNALS 1

4 #include "pulse-db-dbengine.h"

6 -size_t pulse_dbengine_total_memory = 0;

6 +int64_t pulse_dbengine_total_memory = 0;

8 #if defined(ENABLE_DBENGINE)

@@ -660,21 +660,21 @@ void pulse_dbengine_do(bool extended) {

660

661 struct rrdeng_buffer_sizes dbmem = rrdeng_pulse_memory_sizes();

662

663 - size_t buffers_total_size = dbmem.xt_buf + dbmem.wal;

663 + int64_t buffers_total_size = (int64_t)dbmem.xt_buf + (int64_t)dbmem.wal;

664

665 - size_t aral_structures_total_size = 0, aral_used_total_size = 0;

666 - size_t aral_padding_total_size = 0;

665 + int64_t aral_structures_total_size = 0, aral_used_total_size = 0;

666 + int64_t aral_padding_total_size = 0;

667 for(size_t i = 0; i < RRDENG_MEM_MAX ; i++) {

668 - buffers_total_size += aral_free_bytes_from_stats(dbmem.as[i]);

669 - aral_structures_total_size += aral_structures_bytes_from_stats(dbmem.as[i]);

670 - aral_used_total_size += aral_used_bytes_from_stats(dbmem.as[i]);

671 - aral_padding_total_size += aral_padding_bytes_from_stats(dbmem.as[i]);

668 + buffers_total_size += (int64_t)aral_free_bytes_from_stats(dbmem.as[i]);

669 + aral_structures_total_size += (int64_t)aral_structures_bytes_from_stats(dbmem.as[i]);

670 + aral_used_total_size += (int64_t)aral_used_bytes_from_stats(dbmem.as[i]);

671 + aral_padding_total_size += (int64_t)aral_padding_bytes_from_stats(dbmem.as[i]);

672 }

673

674 pulse_dbengine_total_memory =

675 - pgc_main_stats.size + (ssize_t)pgc_open_stats.size + pgc_extent_stats.size +

675 + pgc_main_stats.size + pgc_open_stats.size + pgc_extent_stats.size +

676 mrg_stats.size +

677 - buffers_total_size + aral_structures_total_size + aral_padding_total_size + pgd_padding_bytes();

677 + buffers_total_size + aral_structures_total_size + aral_padding_total_size + (int64_t)pgd_padding_bytes();

678

679 // we need all the above for the total dbengine memory as reported by the non-extended netdata memory chart

680 if(!main_cache || !main_mrg || !extended)

@@ -685,7 +685,7 @@ void pulse_dbengine_do(bool extended) {

685 dbengine2_cache_statistics_charts(&extent_cache_ptrs, &pgc_extent_stats, &pgc_extent_stats_old, "extent", 135300);

686 mrg_get_statistics(main_mrg, &mrg_stats);

687

688 - size_t priority = 135000;

688 + int priority = 135000;

689 {

690 static RRDSET *st_pgc_memory = NULL;

691 static RRDDIM *rd_pgc_memory_main = NULL;

src/daemon/pulse/pulse-db-dbengine.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "daemon/common.h"

8 #if defined(PULSE_INTERNALS)

9 -extern size_t pulse_dbengine_total_memory;

9 +extern int64_t pulse_dbengine_total_memory;

11 #if defined(ENABLE_DBENGINE)

12 void pulse_dbengine_do(bool extended);

src/daemon/pulse/pulse-queries.c

+4 -4

@@ -32,22 +32,22 @@ static struct query_statistics {

32 PAD64(uint64_t) exporters_db_points_read;

33 } query_statistics = { 0 };

35 -void pulse_queries_ml_query_completed(size_t points_read) {

35 +ALWAYS_INLINE void pulse_queries_ml_query_completed(size_t points_read) {

36 __atomic_fetch_add(&query_statistics.ml_queries_made, 1, __ATOMIC_RELAXED);

37 __atomic_fetch_add(&query_statistics.ml_db_points_read, points_read, __ATOMIC_RELAXED);

38 }

40 -void pulse_queries_exporters_query_completed(size_t points_read) {

40 +ALWAYS_INLINE void pulse_queries_exporters_query_completed(size_t points_read) {

41 __atomic_fetch_add(&query_statistics.exporters_queries_made, 1, __ATOMIC_RELAXED);

42 __atomic_fetch_add(&query_statistics.exporters_db_points_read, points_read, __ATOMIC_RELAXED);

43 }

45 -void pulse_queries_backfill_query_completed(size_t points_read) {

45 +ALWAYS_INLINE void pulse_queries_backfill_query_completed(size_t points_read) {

46 __atomic_fetch_add(&query_statistics.backfill_queries_made, 1, __ATOMIC_RELAXED);

47 __atomic_fetch_add(&query_statistics.backfill_db_points_read, points_read, __ATOMIC_RELAXED);

48 }

50 -void pulse_queries_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source) {

50 +ALWAYS_INLINE void pulse_queries_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source) {

51 switch(query_source) {

52 case QUERY_SOURCE_API_DATA:

53 __atomic_fetch_add(&query_statistics.api_data_queries_made, queries, __ATOMIC_RELAXED);

src/database/engine/cache.c

+150 -147

@@ -93,23 +93,23 @@ struct pgc {

93 bool use_all_ram;

95 size_t partitions;

96 - size_t clean_size;

96 + int64_t clean_size;

97 size_t max_dirty_pages_per_call;

98 size_t max_pages_per_inline_eviction;

99 size_t max_skip_pages_per_inline_eviction;

100 size_t max_flushes_inline;

101 size_t max_workers_evict_inline;

102 size_t additional_bytes_per_page;

103 - size_t out_of_memory_protection_bytes;

103 + int64_t out_of_memory_protection_bytes;

104 free_clean_page_callback pgc_free_clean_cb;

105 save_dirty_page_callback pgc_save_dirty_cb;

106 save_dirty_init_callback pgc_save_init_cb;

107 PGC_OPTIONS options;

108

109 - size_t severe_pressure_per1000;

110 - size_t aggressive_evict_per1000;

111 - size_t healthy_size_per1000;

112 - size_t evict_low_threshold_per1000;

109 + ssize_t severe_pressure_per1000;

110 + ssize_t aggressive_evict_per1000;

111 + ssize_t healthy_size_per1000;

112 + ssize_t evict_low_threshold_per1000;

113

114 dynamic_target_cache_size_callback dynamic_target_size_cb;

115 nominal_page_size_callback nominal_page_size_cb;

@@ -130,7 +130,7 @@ struct pgc {

130

131 struct {

132 SPINLOCK spinlock;

133 - size_t per1000;

133 + ssize_t per1000;

134 } usage;

135

136 struct pgc_queue clean; // LRU is applied here to free memory from the cache

@@ -334,21 +334,21 @@ static inline void pgc_size_histogram_del(PGC *cache, struct pgc_size_histogram

334 // ----------------------------------------------------------------------------

335 // evictions control

336

337 -static inline uint64_t pgc_threshold(size_t threshold, uint64_t wanted, uint64_t current, uint64_t clean) {

337 +static ALWAYS_INLINE int64_t pgc_threshold(ssize_t threshold, int64_t wanted, int64_t current, int64_t clean) {

338 if(current < clean)

339 current = clean;

340

341 if(wanted < current - clean)

342 wanted = current - clean;

343

344 - uint64_t ret = wanted * threshold / 1000ULL;

344 + int64_t ret = wanted * threshold / 1000LL;

345 if(ret < current - clean)

346 ret = current - clean;

347

348 return ret;

349 }

350

351 -static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

351 +static ssize_t cache_usage_per1000(PGC *cache, int64_t *size_to_evict) {

352

353 if(size_to_evict)

354 spinlock_lock(&cache->usage.spinlock);

@@ -356,33 +356,33 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

356 else if(!spinlock_trylock(&cache->usage.spinlock))

357 return __atomic_load_n(&cache->usage.per1000, __ATOMIC_RELAXED);

358

359 - uint64_t wanted_cache_size;

359 + int64_t wanted_cache_size;

360

361 - const uint64_t dirty = __atomic_load_n(&cache->dirty.stats->size, __ATOMIC_RELAXED);

362 - const uint64_t hot = __atomic_load_n(&cache->hot.stats->size, __ATOMIC_RELAXED);

363 - const uint64_t clean = __atomic_load_n(&cache->clean.stats->size, __ATOMIC_RELAXED);

364 - const uint64_t evicting = __atomic_load_n(&cache->stats.evicting_size, __ATOMIC_RELAXED);

365 - const uint64_t flushing = __atomic_load_n(&cache->stats.flushing_size, __ATOMIC_RELAXED);

366 - const uint64_t current_cache_size = __atomic_load_n(&cache->stats.size, __ATOMIC_RELAXED);

367 - const uint64_t all_pages_size = hot + dirty + clean + evicting + flushing;

368 - const uint64_t index = current_cache_size > all_pages_size ? current_cache_size - all_pages_size : 0;

369 - const uint64_t referenced_size = __atomic_load_n(&cache->stats.referenced_size, __ATOMIC_RELAXED);

361 + const int64_t dirty = __atomic_load_n(&cache->dirty.stats->size, __ATOMIC_RELAXED);

362 + const int64_t hot = __atomic_load_n(&cache->hot.stats->size, __ATOMIC_RELAXED);

363 + const int64_t clean = __atomic_load_n(&cache->clean.stats->size, __ATOMIC_RELAXED);

364 + const int64_t evicting = __atomic_load_n(&cache->stats.evicting_size, __ATOMIC_RELAXED);

365 + const int64_t flushing = __atomic_load_n(&cache->stats.flushing_size, __ATOMIC_RELAXED);

366 + const int64_t current_cache_size = __atomic_load_n(&cache->stats.size, __ATOMIC_RELAXED);

367 + const int64_t all_pages_size = hot + dirty + clean + evicting + flushing;

368 + const int64_t index = current_cache_size > all_pages_size ? current_cache_size - all_pages_size : 0;

369 + const int64_t referenced_size = __atomic_load_n(&cache->stats.referenced_size, __ATOMIC_RELAXED);

370

371 if(cache->config.options & PGC_OPTIONS_AUTOSCALE) {

372 - const uint64_t dirty_max = __atomic_load_n(&cache->dirty.stats->max_size, __ATOMIC_RELAXED);

373 - const uint64_t hot_max = __atomic_load_n(&cache->hot.stats->max_size, __ATOMIC_RELAXED);

372 + const int64_t dirty_max = __atomic_load_n(&cache->dirty.stats->max_size, __ATOMIC_RELAXED);

373 + const int64_t hot_max = __atomic_load_n(&cache->hot.stats->max_size, __ATOMIC_RELAXED);

374

375 // our promise to users

376 - const uint64_t max_size1 = MAX(hot_max, hot) * 2;

376 + const int64_t max_size1 = MAX(hot_max, hot) * 2;

377

378 // protection against slow flushing

379 - const uint64_t max_size2 = hot_max + ((dirty_max * 2 < hot_max * 2 / 3) ? hot_max * 2 / 3 : dirty_max * 2) + index;

379 + const int64_t max_size2 = hot_max + ((dirty_max * 2 < hot_max * 2 / 3) ? hot_max * 2 / 3 : dirty_max * 2) + index;

380

381 // the final wanted cache size

382 wanted_cache_size = MIN(max_size1, max_size2);

383

384 if(cache->config.dynamic_target_size_cb) {

385 - const uint64_t wanted_cache_size_cb = cache->config.dynamic_target_size_cb();

385 + const int64_t wanted_cache_size_cb = cache->config.dynamic_target_size_cb();

386 if(wanted_cache_size_cb > wanted_cache_size)

387 wanted_cache_size = wanted_cache_size_cb;

388 }

@@ -394,9 +394,9 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

394 wanted_cache_size = hot + dirty + index + cache->config.clean_size;

395

396 // calculate the absolute minimum we can go

397 - const uint64_t min_cache_size1 = (referenced_size > hot ? referenced_size : hot) + dirty + index;

398 - const uint64_t min_cache_size2 = (current_cache_size > clean) ? current_cache_size - clean : min_cache_size1;

399 - const uint64_t min_cache_size = MAX(min_cache_size1, min_cache_size2);

397 + const int64_t min_cache_size1 = (referenced_size > hot ? referenced_size : hot) + dirty + index;

398 + const int64_t min_cache_size2 = (current_cache_size > clean) ? current_cache_size - clean : min_cache_size1;

399 + const int64_t min_cache_size = MAX(min_cache_size1, min_cache_size2);

400

401 if(cache->config.out_of_memory_protection_bytes) {

402 // out of memory protection

@@ -404,10 +404,12 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

404 if(sm.ram_total_bytes) {

405 // when the total exists, ram_available_bytes is also right

406

407 - const uint64_t min_available = cache->config.out_of_memory_protection_bytes;

408 - if (sm.ram_available_bytes < min_available) {

407 + const int64_t ram_available_bytes = (int64_t)sm.ram_available_bytes;

408 +

409 + const int64_t min_available = cache->config.out_of_memory_protection_bytes;

410 + if (ram_available_bytes < min_available) {

411 // we must shrink

410 - uint64_t must_lose = min_available - sm.ram_available_bytes;

412 + int64_t must_lose = min_available - ram_available_bytes;

413

414 if(current_cache_size > must_lose)

415 wanted_cache_size = current_cache_size - must_lose;

@@ -416,7 +418,7 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

418 }

419 else if(cache->config.use_all_ram) {

420 // we can grow

419 - wanted_cache_size = current_cache_size + (sm.ram_available_bytes - min_available);

421 + wanted_cache_size = current_cache_size + (ram_available_bytes - min_available);

422 }

423 }

424 }

@@ -429,21 +431,21 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

431 if(unlikely(wanted_cache_size < 65536))

432 wanted_cache_size = 65536;

433

432 - const size_t per1000 = (size_t)(current_cache_size * 1000ULL / wanted_cache_size);

434 + const ssize_t per1000 = (ssize_t)(current_cache_size * 1000LL / wanted_cache_size);

435 __atomic_store_n(&cache->usage.per1000, per1000, __ATOMIC_RELAXED);

436 __atomic_store_n(&cache->stats.wanted_cache_size, wanted_cache_size, __ATOMIC_RELAXED);

437 __atomic_store_n(&cache->stats.current_cache_size, current_cache_size, __ATOMIC_RELAXED);

438

437 - uint64_t healthy_target = pgc_threshold(cache->config.healthy_size_per1000, wanted_cache_size, current_cache_size, clean);

439 + int64_t healthy_target = pgc_threshold(cache->config.healthy_size_per1000, wanted_cache_size, current_cache_size, clean);

440 if(current_cache_size > healthy_target) {

439 - uint64_t low_watermark_target = pgc_threshold(cache->config.evict_low_threshold_per1000, wanted_cache_size, current_cache_size, clean);

441 + int64_t low_watermark_target = pgc_threshold(cache->config.evict_low_threshold_per1000, wanted_cache_size, current_cache_size, clean);

442

441 - uint64_t size_to_evict_now = current_cache_size - low_watermark_target;

443 + int64_t size_to_evict_now = current_cache_size - low_watermark_target;

444 if(size_to_evict_now > clean)

445 size_to_evict_now = clean;

446

447 if(size_to_evict)

446 - *size_to_evict = (size_t)size_to_evict_now;

448 + *size_to_evict = size_to_evict_now;

449

450 bool signal = false;

451 if(per1000 >= cache->config.severe_pressure_per1000) {

@@ -466,7 +468,7 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

468 return per1000;

469 }

470

469 -static inline bool cache_pressure(PGC *cache, size_t limit) {

471 +static inline bool cache_pressure(PGC *cache, ssize_t limit) {

472 return (cache_usage_per1000(cache, NULL) >= limit);

473 }

474

@@ -481,8 +483,8 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic

483 static inline bool flushing_critical(PGC *cache);

484 static bool flush_pages(PGC *cache, size_t max_flushes, Word_t section, bool wait, bool all_of_them);

485

484 -static void evict_pages_inline(PGC *cache, bool on_release) {

485 - const size_t per1000 = cache_usage_per1000(cache, NULL);

486 +static ALWAYS_INLINE void evict_pages_inline(PGC *cache, bool on_release) {

487 + const ssize_t per1000 = cache_usage_per1000(cache, NULL);

488

489 if(!(cache->config.options & PGC_OPTIONS_EVICT_PAGES_NO_INLINE)) {

490 if (per1000 > cache->config.aggressive_evict_per1000 && !on_release) {

@@ -505,15 +507,15 @@ static void evict_pages_inline(PGC *cache, bool on_release) {

507 }

508 }

509

508 -static inline void evict_on_clean_page_added(PGC *cache) {

510 +static ALWAYS_INLINE void evict_on_clean_page_added(PGC *cache) {

511 evict_pages_inline(cache, false);

512 }

513

512 -static inline void evict_on_page_release_when_permitted(PGC *cache) {

514 +static ALWAYS_INLINE void evict_on_page_release_when_permitted(PGC *cache) {

515 evict_pages_inline(cache, true);

516 }

517

516 -static inline void flush_inline(PGC *cache, bool on_release) {

518 +static ALWAYS_INLINE void flush_inline(PGC *cache, bool on_release) {

519 if(!(cache->config.options & PGC_OPTIONS_FLUSH_PAGES_NO_INLINE) && flushing_critical(cache)) {

520 if (on_release)

521 p2_add_fetch(&cache->stats.p2_waste_flush_on_release, 1);

@@ -524,11 +526,11 @@ static inline void flush_inline(PGC *cache, bool on_release) {

526 }

527 }

528

527 -static inline void flush_on_page_add(PGC *cache) {

529 +static ALWAYS_INLINE void flush_on_page_add(PGC *cache) {

530 flush_inline(cache, false);

531 }

532

531 -static inline void flush_on_page_hot_release(PGC *cache) {

533 +static ALWAYS_INLINE void flush_on_page_hot_release(PGC *cache) {

534 flush_inline(cache, true);

535 }

536

@@ -536,8 +538,6 @@ static inline void flush_on_page_hot_release(PGC *cache) {

538 // ----------------------------------------------------------------------------

539 // flushing control

540

539 -static bool flush_pages(PGC *cache, size_t max_flushes, Word_t section, bool wait, bool all_of_them);

540 -

541 static inline bool flushing_critical(PGC *cache) {

542 if(unlikely(__atomic_load_n(&cache->dirty.stats->size, __ATOMIC_RELAXED) > __atomic_load_n(&cache->hot.stats->max_size, __ATOMIC_RELAXED))) {

543 __atomic_add_fetch(&cache->stats.events_flush_critical, 1, __ATOMIC_RELAXED);

@@ -550,11 +550,25 @@ static inline bool flushing_critical(PGC *cache) {

550 // ----------------------------------------------------------------------------

551 // Linked list management

552

553 -static inline void atomic_set_max(size_t *max, size_t desired) {

553 +static inline void atomic_set_max_size_t(size_t *max, size_t desired) {

554 size_t expected;

555

556 expected = __atomic_load_n(max, __ATOMIC_RELAXED);

557

558 + do {

559 +

560 + if(expected >= desired)

561 + return;

562 +

563 + } while(!__atomic_compare_exchange_n(max, &expected, desired,

564 + false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));

565 +}

566 +

567 +static inline void atomic_set_max_int64_t(int64_t *max, int64_t desired) {

568 + int64_t expected;

569 +

570 + expected = __atomic_load_n(max, __ATOMIC_RELAXED);

571 +

572 do {

573

574 if(expected >= desired)

@@ -591,25 +605,13 @@ static void pgc_section_pages_static_aral_init(void) {

605 spinlock_unlock(&spinlock);

606 }

607

594 -static ALWAYS_INLINE void

595 -pgc_stats_queue_judy_change(PGC *cache, struct pgc_queue *ll, size_t mem_before_judyl, size_t mem_after_judyl) {

596 - if(mem_after_judyl > mem_before_judyl) {

597 - __atomic_add_fetch(&ll->stats->size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED);

598 - __atomic_add_fetch(&cache->stats.size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED);

599 - }

600 - else if(mem_after_judyl < mem_before_judyl) {

601 - __atomic_sub_fetch(&ll->stats->size, mem_before_judyl - mem_after_judyl, __ATOMIC_RELAXED);

602 - __atomic_sub_fetch(&cache->stats.size, mem_before_judyl - mem_after_judyl, __ATOMIC_RELAXED);

603 - }

608 +static ALWAYS_INLINE void pgc_stats_queue_judy_change(PGC *cache, struct pgc_queue *ll, int64_t delta) {

609 + __atomic_add_fetch(&ll->stats->size, delta, __ATOMIC_RELAXED);

610 + __atomic_add_fetch(&cache->stats.size, delta, __ATOMIC_RELAXED);

611 }

612

606 -static ALWAYS_INLINE void pgc_stats_index_judy_change(PGC *cache, size_t mem_before_judyl, size_t mem_after_judyl) {

607 - if(mem_after_judyl > mem_before_judyl) {

608 - __atomic_add_fetch(&cache->stats.size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED);

609 - }

610 - else if(mem_after_judyl < mem_before_judyl) {

611 - __atomic_sub_fetch(&cache->stats.size, mem_before_judyl - mem_after_judyl, __ATOMIC_RELAXED);

612 - }

613 +static ALWAYS_INLINE void pgc_stats_index_judy_change(PGC *cache, int64_t delta) {

614 + __atomic_add_fetch(&cache->stats.size, delta, __ATOMIC_RELAXED);

615 }

616

617 static ALWAYS_INLINE void pgc_queue_add(PGC *cache __maybe_unused, struct pgc_queue *q, PGC_PAGE *page, bool having_lock, WAITQ_PRIORITY prio __maybe_unused) {

@@ -624,11 +626,10 @@ static ALWAYS_INLINE void pgc_queue_add(PGC *cache __maybe_unused, struct pgc_qu

626 if(q->linked_list_in_sections_judy) {

627 // HOT and DIRTY pages end up here.

628

627 - size_t mem_before_judyl, mem_after_judyl;

629 + JudyAllocThreadPulseReset();

630 + int64_t mem_delta = 0;

631

629 - mem_before_judyl = JudyLMemUsed(q->sections_judy);

632 Pvoid_t *section_pages_pptr = JudyLIns(&q->sections_judy, page->section, PJE0);

631 - mem_after_judyl = JudyLMemUsed(q->sections_judy);

633

634 struct section_pages *sp = *section_pages_pptr;

635 if(!sp) {

@@ -638,9 +639,11 @@ static ALWAYS_INLINE void pgc_queue_add(PGC *cache __maybe_unused, struct pgc_qu

639

640 *section_pages_pptr = sp;

641

641 - mem_after_judyl += sizeof(struct section_pages);

642 + mem_delta += sizeof(struct section_pages);

643 }

643 - pgc_stats_queue_judy_change(cache, q, mem_before_judyl, mem_after_judyl);

644 +

645 + mem_delta += JudyAllocThreadPulseGetAndReset();

646 + pgc_stats_queue_judy_change(cache, q, mem_delta);

647

648 sp->entries++;

649 sp->size += page->assumed_size;

@@ -670,12 +673,12 @@ static ALWAYS_INLINE void pgc_queue_add(PGC *cache __maybe_unused, struct pgc_qu

673 pgc_queue_unlock(cache, q);

674

675 size_t entries = __atomic_add_fetch(&q->stats->entries, 1, __ATOMIC_RELAXED);

673 - size_t size = __atomic_add_fetch(&q->stats->size, page->assumed_size, __ATOMIC_RELAXED);

676 + int64_t size = __atomic_add_fetch(&q->stats->size, page->assumed_size, __ATOMIC_RELAXED);

677 __atomic_add_fetch(&q->stats->added_entries, 1, __ATOMIC_RELAXED);

678 __atomic_add_fetch(&q->stats->added_size, page->assumed_size, __ATOMIC_RELAXED);

679

677 - atomic_set_max(&q->stats->max_entries, entries);

678 - atomic_set_max(&q->stats->max_size, size);

680 + atomic_set_max_size_t(&q->stats->max_entries, entries);

681 + atomic_set_max_int64_t(&q->stats->max_size, size);

682

683 if(cache->config.stats)

684 pgc_size_histogram_add(cache, &q->stats->size_histogram, page);

@@ -711,19 +714,21 @@ static ALWAYS_INLINE void pgc_queue_del(PGC *cache __maybe_unused, struct pgc_qu

714 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(sp->base, page, link.prev, link.next);

715

716 if(!sp->base) {

714 - size_t mem_before_judyl, mem_after_judyl;

717 + JudyAllocThreadPulseReset();

718 + int64_t mem_delta = 0;

719

716 - mem_before_judyl = JudyLMemUsed(q->sections_judy);

720 int rc = JudyLDel(&q->sections_judy, page->section, PJE0);

718 - mem_after_judyl = JudyLMemUsed(q->sections_judy);

721

722 if(!rc)

723 fatal("DBENGINE CACHE: cannot delete section from Judy LL");

724

725 // freez(sp);

726 aral_freez(pgc_sections_aral, sp);

725 - mem_after_judyl -= sizeof(struct section_pages);

726 - pgc_stats_queue_judy_change(cache, q, mem_before_judyl, mem_after_judyl);

727 +

728 + mem_delta -= sizeof(struct section_pages);

729 + mem_delta += JudyAllocThreadPulseGetAndReset();

730 +

731 + pgc_stats_queue_judy_change(cache, q, mem_delta);

732 }

733 }

734 else {

@@ -860,7 +865,7 @@ static ALWAYS_INLINE void PGC_REFERENCED_PAGES_PLUS1(PGC *cache, PGC_PAGE *page)

865 __atomic_add_fetch(&cache->stats.referenced_size, page->assumed_size, __ATOMIC_RELAXED);

866 }

867

863 -static ALWAYS_INLINE void PGC_REFERENCED_PAGES_MINUS1(PGC *cache, size_t assumed_size) {

868 +static ALWAYS_INLINE void PGC_REFERENCED_PAGES_MINUS1(PGC *cache, int64_t assumed_size) {

869 __atomic_sub_fetch(&cache->stats.referenced_entries, 1, __ATOMIC_RELAXED);

870 __atomic_sub_fetch(&cache->stats.referenced_size, assumed_size, __ATOMIC_RELAXED);

871 }

@@ -885,7 +890,7 @@ static ALWAYS_INLINE bool page_acquire(PGC *cache, PGC_PAGE *page) {

890 static ALWAYS_INLINE void page_release(PGC *cache, PGC_PAGE *page, bool evict_if_necessary) {

891 __atomic_add_fetch(&cache->stats.releases, 1, __ATOMIC_RELAXED);

892

888 - size_t assumed_size = page->assumed_size; // take the size before we release it

893 + int64_t assumed_size = page->assumed_size; // take the size before we release it

894

895 if(refcount_release(&page->refcount) == 0) {

896 PGC_REFERENCED_PAGES_MINUS1(cache, assumed_size);

@@ -917,7 +922,7 @@ static ALWAYS_INLINE bool non_acquired_page_get_for_deletion___while_having_clea

922 static ALWAYS_INLINE bool acquired_page_get_for_deletion_or_release_it(PGC *cache __maybe_unused, PGC_PAGE *page) {

923 __atomic_add_fetch(&cache->stats.acquires_for_deletion, 1, __ATOMIC_RELAXED);

924

920 - size_t assumed_size = page->assumed_size; // take the size before we release it

925 + int64_t assumed_size = page->assumed_size; // take the size before we release it

926

927 if(refcount_release_and_acquire_for_deletion(&page->refcount)) {

928 PGC_REFERENCED_PAGES_MINUS1(cache, assumed_size);

@@ -1008,26 +1013,20 @@ static void remove_this_page_from_index_unsafe(PGC *cache, PGC_PAGE *page, size_

1013 fatal("DBENGINE CACHE: page with start time '%ld' of metric '%lu' in section '%lu' should exist, but the index returned a different address.",

1014 page->start_time_s, page->metric_id, page->section);

1015

1011 - size_t mem_before_judyl = 0, mem_after_judyl = 0;

1016 + JudyAllocThreadPulseReset();

1017

1013 - mem_before_judyl += JudyLMemUsed(*pages_judy_pptr);

1018 if(unlikely(!JudyLDel(pages_judy_pptr, page->start_time_s, PJE0)))

1019 fatal("DBENGINE CACHE: page with start time '%ld' of metric '%lu' in section '%lu' exists, but cannot be deleted.",

1020 page->start_time_s, page->metric_id, page->section);

1017 - mem_after_judyl += JudyLMemUsed(*pages_judy_pptr);

1021

1019 - mem_before_judyl += JudyLMemUsed(*metrics_judy_pptr);

1022 if(!*pages_judy_pptr && !JudyLDel(metrics_judy_pptr, page->metric_id, PJE0))

1023 fatal("DBENGINE CACHE: metric '%lu' in section '%lu' exists and is empty, but cannot be deleted.",

1024 page->metric_id, page->section);

1023 - mem_after_judyl += JudyLMemUsed(*metrics_judy_pptr);

1025

1025 - mem_before_judyl += JudyLMemUsed(cache->index[partition].sections_judy);

1026 if(!*metrics_judy_pptr && !JudyLDel(&cache->index[partition].sections_judy, page->section, PJE0))

1027 fatal("DBENGINE CACHE: section '%lu' exists and is empty, but cannot be deleted.", page->section);

1028 - mem_after_judyl += JudyLMemUsed(cache->index[partition].sections_judy);

1028

1030 - pgc_stats_index_judy_change(cache, mem_before_judyl, mem_after_judyl);

1029 + pgc_stats_index_judy_change(cache, JudyAllocThreadPulseGetAndReset());

1030

1031 pointer_del(cache, page);

1032 }

@@ -1067,7 +1066,7 @@ static inline bool make_acquired_page_clean_and_evict_or_page_release(PGC *cache

1066

1067 // returns true, when there is potentially more work to do

1068 static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evict, bool wait, bool all_of_them, evict_filter filter, void *data) {

1070 - size_t per1000 = cache_usage_per1000(cache, NULL);

1069 + ssize_t per1000 = cache_usage_per1000(cache, NULL);

1070

1071 if(!all_of_them && per1000 < cache->config.healthy_size_per1000)

1072 // don't bother - not enough to do anything

@@ -1101,7 +1100,7 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic

1100 size_t max_pages_to_evict = 0;

1101

1102 do {

1104 - size_t max_size_to_evict = 0;

1103 + int64_t max_size_to_evict = 0;

1104 if (unlikely(all_of_them)) {

1105 // evict them all

1106 max_size_to_evict = SIZE_MAX;

@@ -1165,7 +1164,7 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic

1164

1165 // find a page to evict

1166 PGC_PAGE *pages_to_evict = NULL;

1168 - size_t pages_to_evict_size = 0;

1167 + int64_t pages_to_evict_size = 0;

1168 size_t pages_to_evict_count = 0;

1169 for(PGC_PAGE *page = cache->clean.base, *next = NULL, *first_page_we_relocated = NULL; page ; page = next) {

1170 next = page->link.next;

@@ -1292,7 +1291,7 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic

1291

1292 timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_LOOP);

1293

1295 - size_t page_size = page->assumed_size;

1294 + int64_t page_size = page->assumed_size;

1295 free_this_page(cache, page, partition);

1296

1297 timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_PAGE);

@@ -1313,7 +1312,7 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic

1312 // just one page to be evicted

1313 PGC_PAGE *page = pages_to_evict;

1314

1316 - size_t page_size = page->assumed_size;

1315 + int64_t page_size = page->assumed_size;

1316

1317 size_t partition = pgc_indexing_partition(cache, page->metric_id);

1318 pgc_index_write_lock(cache, partition);

@@ -1354,7 +1353,7 @@ premature_exit:

1353 return stopped_before_finishing;

1354 }

1355

1357 -static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {

1356 +static PGC_PAGE *pgc_page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {

1357 internal_fatal(entry->start_time_s < 0 || entry->end_time_s < 0,

1358 "DBENGINE CACHE: timestamps are negative");

1359

@@ -1364,7 +1363,31 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {

1363

1364 #ifdef PGC_WITH_ARAL

1365 PGC_PAGE *allocation = aral_mallocz(cache->index[partition].aral);

1366 +#else

1367 + PGC_PAGE *allocation = mallocz(sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page);

1368 #endif

1369 +

1370 + allocation->refcount = 1;

1371 + allocation->accesses = (entry->hot) ? 0 : 1;

1372 + allocation->flags = 0;

1373 + allocation->section = entry->section;

1374 + allocation->metric_id = entry->metric_id;

1375 + allocation->start_time_s = entry->start_time_s;

1376 + allocation->end_time_s = entry->end_time_s,

1377 + allocation->update_every_s = entry->update_every_s,

1378 + allocation->data = entry->data;

1379 + allocation->assumed_size = page_assumed_size(cache, entry->size);

1380 + spinlock_init(&allocation->transition_spinlock);

1381 + allocation->link.prev = NULL;

1382 + allocation->link.next = NULL;

1383 +

1384 + if(cache->config.additional_bytes_per_page) {

1385 + if(entry->custom_data)

1386 + memcpy(allocation->custom_data, entry->custom_data, cache->config.additional_bytes_per_page);

1387 + else

1388 + memset(allocation->custom_data, 0, cache->config.additional_bytes_per_page);

1389 + }

1390 +

1391 PGC_PAGE *page;

1392 size_t spins = 0;

1393

@@ -1379,58 +1402,29 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {

1402

1403 pgc_index_write_lock(cache, partition);

1404

1382 - size_t mem_before_judyl = 0, mem_after_judyl = 0;

1405 + JudyAllocThreadPulseReset();

1406

1384 - mem_before_judyl += JudyLMemUsed(cache->index[partition].sections_judy);

1407 Pvoid_t *metrics_judy_pptr = JudyLIns(&cache->index[partition].sections_judy, entry->section, PJE0);

1408 if(unlikely(!metrics_judy_pptr || metrics_judy_pptr == PJERR))

1409 fatal("DBENGINE CACHE: corrupted sections judy array");

1388 - mem_after_judyl += JudyLMemUsed(cache->index[partition].sections_judy);

1410

1390 - mem_before_judyl += JudyLMemUsed(*metrics_judy_pptr);

1411 Pvoid_t *pages_judy_pptr = JudyLIns(metrics_judy_pptr, entry->metric_id, PJE0);

1412 if(unlikely(!pages_judy_pptr || pages_judy_pptr == PJERR))

1413 fatal("DBENGINE CACHE: corrupted pages judy array");

1394 - mem_after_judyl += JudyLMemUsed(*metrics_judy_pptr);

1414

1396 - mem_before_judyl += JudyLMemUsed(*pages_judy_pptr);

1415 Pvoid_t *page_ptr = JudyLIns(pages_judy_pptr, entry->start_time_s, PJE0);

1416 if(unlikely(!page_ptr || page_ptr == PJERR))

1417 fatal("DBENGINE CACHE: corrupted page in judy array");

1400 - mem_after_judyl += JudyLMemUsed(*pages_judy_pptr);

1418

1402 - pgc_stats_index_judy_change(cache, mem_before_judyl, mem_after_judyl);

1419 + pgc_stats_index_judy_change(cache, JudyAllocThreadPulseGetAndReset());

1420

1421 page = *page_ptr;

1422

1423 if (likely(!page)) {

1407 -#ifdef PGC_WITH_ARAL

1424 + // consume it

1425 page = allocation;

1426 allocation = NULL;

1410 -#else

1411 - page = mallocz(sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page);

1412 -#endif

1413 - page->refcount = 1;

1414 - page->accesses = (entry->hot) ? 0 : 1;

1415 - page->flags = 0;

1416 - page->section = entry->section;

1417 - page->metric_id = entry->metric_id;

1418 - page->start_time_s = entry->start_time_s;

1419 - page->end_time_s = entry->end_time_s,

1420 - page->update_every_s = entry->update_every_s,

1421 - page->data = entry->data;

1422 - page->assumed_size = page_assumed_size(cache, entry->size);

1423 - spinlock_init(&page->transition_spinlock);

1424 - page->link.prev = NULL;

1425 - page->link.next = NULL;

1426 -

1427 - if(cache->config.additional_bytes_per_page) {

1428 - if(entry->custom_data)

1429 - memcpy(page->custom_data, entry->custom_data, cache->config.additional_bytes_per_page);

1430 - else

1431 - memset(page->custom_data, 0, cache->config.additional_bytes_per_page);

1432 - }

1433 -

1427 +

1428 // put it in the index

1429 *page_ptr = page;

1430 pointer_add(cache, page);

@@ -1471,10 +1465,13 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {

1465

1466 } while(!page);

1467

1468 + if(allocation) {

1469 #ifdef PGC_WITH_ARAL

1475 - if(allocation)

1470 aral_freez(cache->index[partition].aral, allocation);

1471 +#else

1472 + freez(allocation);

1473 #endif

1474 + }

1475

1476 if(spins > 1)

1477 p2_add_fetch(&cache->stats.p2_waste_insert_spins, spins - 1);

@@ -1734,10 +1731,16 @@ static bool flush_pages(PGC *cache, size_t max_flushes, Word_t section, bool wai

1731

1732 PGC_ENTRY array[optimal_flush_size];

1733 PGC_PAGE *pages[optimal_flush_size];

1737 - size_t pages_added = 0, pages_added_size = 0;

1738 - size_t pages_removed_dirty = 0, pages_removed_dirty_size = 0;

1739 - size_t pages_cancelled = 0, pages_cancelled_size = 0;

1740 - size_t pages_made_clean = 0, pages_made_clean_size = 0;

1734 +

1735 + size_t pages_added = 0,

1736 + pages_removed_dirty = 0,

1737 + pages_cancelled = 0,

1738 + pages_made_clean = 0;

1739 +

1740 + int64_t pages_added_size = 0,

1741 + pages_removed_dirty_size = 0,

1742 + pages_cancelled_size = 0,

1743 + pages_made_clean_size = 0;

1744

1745 PGC_PAGE *page = sp->base;

1746 while (page && pages_added < optimal_flush_size) {

@@ -1924,7 +1927,7 @@ static void *pgc_evict_thread(void *ptr) {

1927 if (nd_thread_signaled_to_cancel())

1928 break;

1929

1927 - size_t size_to_evict = 0;

1930 + int64_t size_to_evict = 0;

1931 bool system_cleanup = false;

1932 if(cache_usage_per1000(cache, &size_to_evict) > cache->config.aggressive_evict_per1000)

1933 system_cleanup = true;

@@ -1986,7 +1989,7 @@ PGC *pgc_create(const char *name,

1989 cache->config.pgc_save_dirty_cb = pgc_save_dirty_cb;

1990

1991 // eviction strategy

1989 - cache->config.clean_size = (clean_size_bytes < 1 * 1024 * 1024) ? 1 * 1024 * 1024 : clean_size_bytes;

1992 + cache->config.clean_size = (clean_size_bytes < 1 * 1024 * 1024) ? 1 * 1024 * 1024 : (int64_t)clean_size_bytes;

1993 cache->config.pgc_free_clean_cb = pgc_free_cb;

1994 cache->config.max_workers_evict_inline = max_inline_evictors;

1995 cache->config.max_pages_per_inline_eviction = max_pages_per_inline_eviction;

@@ -2000,7 +2003,7 @@ PGC *pgc_create(const char *name,

2003

2004 // use all ram and protection from out of memory

2005 cache->config.use_all_ram = dbengine_use_all_ram_for_caches;

2003 - cache->config.out_of_memory_protection_bytes = dbengine_out_of_memory_protection;

2006 + cache->config.out_of_memory_protection_bytes = (int64_t)dbengine_out_of_memory_protection;

2007

2008 // partitions

2009 if(partitions == 0) partitions = netdata_conf_cpus();

@@ -2115,11 +2118,11 @@ void pgc_destroy(PGC *cache) {

2118 }

2119 }

2120

2118 -PGC_PAGE *pgc_page_add_and_acquire(PGC *cache, PGC_ENTRY entry, bool *added) {

2119 - return page_add(cache, &entry, added);

2121 +ALWAYS_INLINE PGC_PAGE *pgc_page_add_and_acquire(PGC *cache, PGC_ENTRY entry, bool *added) {

2122 + return pgc_page_add(cache, &entry, added);

2123 }

2124

2122 -PGC_PAGE *pgc_page_dup(PGC *cache, PGC_PAGE *page) {

2125 +ALWAYS_INLINE PGC_PAGE *pgc_page_dup(PGC *cache, PGC_PAGE *page) {

2126 if(!page_acquire(cache, page))

2127 fatal("DBENGINE CACHE: tried to dup a page that is not acquired!");

2128

@@ -2130,7 +2133,7 @@ ALWAYS_INLINE void pgc_page_release(PGC *cache, PGC_PAGE *page) {

2133 page_release(cache, page, is_page_clean(page));

2134 }

2135

2133 -void pgc_page_hot_to_dirty_and_release(PGC *cache, PGC_PAGE *page, bool never_flush) {

2136 +ALWAYS_INLINE void pgc_page_hot_to_dirty_and_release(PGC *cache, PGC_PAGE *page, bool never_flush) {

2137 p2_add_fetch(&cache->stats.p2_workers_hot2dirty, 1);

2138

2139 //#ifdef NETDATA_INTERNAL_CHECKS

@@ -2240,12 +2243,12 @@ bool pgc_is_page_clean(PGC_PAGE *page) {

2243

2244 void pgc_reset_hot_max(PGC *cache) {

2245 size_t entries = __atomic_load_n(&cache->hot.stats->entries, __ATOMIC_RELAXED);

2243 - size_t size = __atomic_load_n(&cache->hot.stats->size, __ATOMIC_RELAXED);

2246 + int64_t size = __atomic_load_n(&cache->hot.stats->size, __ATOMIC_RELAXED);

2247

2248 __atomic_store_n(&cache->hot.stats->max_entries, entries, __ATOMIC_RELAXED);

2249 __atomic_store_n(&cache->hot.stats->max_size, size, __ATOMIC_RELAXED);

2250

2248 - size_t size_to_evict = 0;

2251 + int64_t size_to_evict = 0;

2252 cache_usage_per1000(cache, &size_to_evict);

2253 evict_pages(cache, 0, 0, true, false);

2254 }

@@ -2255,7 +2258,7 @@ void pgc_set_dynamic_target_cache_size_callback(PGC *cache, dynamic_target_cache

2258 cache->config.out_of_memory_protection_bytes = 0;

2259 cache->config.use_all_ram = false;

2260

2258 - size_t size_to_evict = 0;

2261 + int64_t size_to_evict = 0;

2262 cache_usage_per1000(cache, &size_to_evict);

2263 evict_pages(cache, 0, 0, true, false);

2264 }

@@ -2264,11 +2267,11 @@ void pgc_set_nominal_page_size_callback(PGC *cache, nominal_page_size_callback c

2267 cache->config.nominal_page_size_cb = callback;

2268 }

2269

2267 -size_t pgc_get_current_cache_size(PGC *cache) {

2270 +int64_t pgc_get_current_cache_size(PGC *cache) {

2271 return __atomic_load_n(&cache->stats.current_cache_size, __ATOMIC_RELAXED);

2272 }

2273

2271 -size_t pgc_get_wanted_cache_size(PGC *cache) {

2274 +int64_t pgc_get_wanted_cache_size(PGC *cache) {

2275 return __atomic_load_n(&cache->stats.wanted_cache_size, __ATOMIC_RELAXED);

2276 }

2277

@@ -2307,13 +2310,13 @@ void pgc_page_hot_set_end_time_s(PGC *cache __maybe_unused, PGC_PAGE *page, time

2310 if(queue_stats && cache->config.stats)

2311 pgc_size_histogram_del(cache, &queue_stats->size_histogram, page);

2312

2310 - size_t old_assumed_size = page->assumed_size;

2313 + int64_t old_assumed_size = page->assumed_size;

2314

2315 size_t old_size = page_size_from_assumed_size(cache, old_assumed_size);

2316 size_t size = old_size + additional_bytes;

2317 page->assumed_size = page_assumed_size(cache, size);

2318

2316 - size_t delta = page->assumed_size - old_assumed_size;

2319 + int64_t delta = page->assumed_size - old_assumed_size;

2320 __atomic_add_fetch(&cache->stats.size, delta, __ATOMIC_RELAXED);

2321 __atomic_add_fetch(&cache->stats.added_size, delta, __ATOMIC_RELAXED);

2322 __atomic_add_fetch(&cache->stats.referenced_size, delta, __ATOMIC_RELAXED);

src/database/engine/cache.h

+18 -18

@@ -51,36 +51,36 @@ struct pgc_queue_statistics {

51 struct pgc_size_histogram size_histogram;

53 PAD64(size_t) entries;

54 - PAD64(size_t) size;

54 + PAD64(int64_t) size;

56 PAD64(size_t) max_entries;

57 - PAD64(size_t) max_size;

57 + PAD64(int64_t) max_size;

59 PAD64(size_t) added_entries;

60 - PAD64(size_t) added_size;

60 + PAD64(int64_t) added_size;

62 PAD64(size_t) removed_entries;

63 - PAD64(size_t) removed_size;

63 + PAD64(int64_t) removed_size;

64 };

66 struct pgc_statistics {

67 - PAD64(size_t) wanted_cache_size;

68 - PAD64(size_t) current_cache_size;

67 + PAD64(int64_t) wanted_cache_size;

68 + PAD64(int64_t) current_cache_size;

70 // ----------------------------------------------------------------------------------------------------------------

71 // volume

73 PAD64(size_t) entries; // all the entries (includes clean, dirty, hot)

74 - PAD64(size_t) size; // all the entries (includes clean, dirty, hot)

74 + PAD64(int64_t) size; // all the entries (includes clean, dirty, hot)

76 PAD64(size_t) referenced_entries; // all the entries currently referenced

77 - PAD64(size_t) referenced_size; // all the entries currently referenced

77 + PAD64(int64_t) referenced_size; // all the entries currently referenced

79 PAD64(size_t) added_entries;

80 - PAD64(size_t) added_size;

80 + PAD64(int64_t) added_size;

82 PAD64(size_t) removed_entries;

83 - PAD64(size_t) removed_size;

83 + PAD64(int64_t) removed_size;

85 #ifdef PGC_COUNT_POINTS_COLLECTED

86 PAD64(size_t) points_collected;

@@ -90,13 +90,13 @@ struct pgc_statistics {

90 // migrations

92 PAD64(size_t) evicting_entries;

93 - PAD64(size_t) evicting_size;

93 + PAD64(int64_t) evicting_size;

95 PAD64(size_t) flushing_entries;

96 - PAD64(size_t) flushing_size;

96 + PAD64(int64_t) flushing_size;

98 PAD64(size_t) hot2dirty_entries;

99 - PAD64(size_t) hot2dirty_size;

99 + PAD64(int64_t) hot2dirty_size;

100

101 PAD64(size_t) hot_empty_pages_evicted_immediately;

102 PAD64(size_t) hot_empty_pages_evicted_later;

@@ -118,8 +118,8 @@ struct pgc_statistics {

118 PAD64(size_t) searches_closest_misses;

119

120 PAD64(size_t) flushes_completed;

121 - PAD64(size_t) flushes_completed_size;

122 - PAD64(size_t) flushes_cancelled_size;

121 + PAD64(int64_t) flushes_completed_size;

122 + PAD64(int64_t) flushes_cancelled_size;

123

124 // ----------------------------------------------------------------------------------------------------------------

125 // critical events

@@ -219,8 +219,8 @@ bool pgc_is_page_hot(PGC_PAGE *page);

219 bool pgc_is_page_dirty(PGC_PAGE *page);

220 bool pgc_is_page_clean(PGC_PAGE *page);

221 void pgc_reset_hot_max(PGC *cache);

222 -size_t pgc_get_current_cache_size(PGC *cache);

223 -size_t pgc_get_wanted_cache_size(PGC *cache);

222 +int64_t pgc_get_current_cache_size(PGC *cache);

223 +int64_t pgc_get_wanted_cache_size(PGC *cache);

224

225 // resetting the end time of a hot page

226 void pgc_page_hot_set_end_time_s(PGC *cache, PGC_PAGE *page, time_t end_time_s, size_t additional_bytes);

@@ -232,7 +232,7 @@ void pgc_open_evict_clean_pages_of_datafile(PGC *cache, struct rrdengine_datafil

232 size_t pgc_count_clean_pages_having_data_ptr(PGC *cache, Word_t section, void *ptr);

233 size_t pgc_count_hot_pages_having_data_ptr(PGC *cache, Word_t section, void *ptr);

234

235 -typedef size_t (*dynamic_target_cache_size_callback)(void);

235 +typedef int64_t (*dynamic_target_cache_size_callback)(void);

236 void pgc_set_dynamic_target_cache_size_callback(PGC *cache, dynamic_target_cache_size_callback callback);

237

238 typedef size_t (*nominal_page_size_callback)(void *);

src/database/engine/datafile.c

+1 -1

@@ -38,7 +38,7 @@ static struct rrdengine_datafile *datafile_alloc_and_init(struct rrdengine_insta

38 return datafile;

39 }

41 -bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason) {

41 +ALWAYS_INLINE bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason) {

42 bool ret;

44 spinlock_lock(&df->users.spinlock);

src/database/engine/journalfile.c

+3 -3

@@ -73,7 +73,7 @@ void journalfile_v1_generate_path(struct rrdengine_datafile *datafile, char *str

74 // ----------------------------------------------------------------------------

76 -struct rrdengine_datafile *njfv2idx_find_and_acquire_j2_header(NJFV2IDX_FIND_STATE *s) {

76 +ALWAYS_INLINE struct rrdengine_datafile *njfv2idx_find_and_acquire_j2_header(NJFV2IDX_FIND_STATE *s) {

77 struct rrdengine_datafile *datafile = NULL;

79 rw_spinlock_read_lock(&s->ctx->njfv2idx.spinlock);

@@ -330,7 +330,7 @@ void journalfile_v2_data_unmount_cleanup(time_t now_s) {

330 }

331 }

332

333 -struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s) {

333 +ALWAYS_INLINE struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s) {

334 spinlock_lock(&journalfile->v2.spinlock);

335

336 bool has_data = (journalfile->v2.flags & JOURNALFILE_FLAG_IS_AVAILABLE);

@@ -361,7 +361,7 @@ struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfi

361 return NULL;

362 }

363

364 -void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile) {

364 +ALWAYS_INLINE void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile) {

365 spinlock_lock(&journalfile->v2.spinlock);

366

367 internal_fatal(!journalfile->mmap.data, "trying to release a journalfile without data");

src/database/engine/metric.c

+35 -35

@@ -94,7 +94,7 @@ static inline void mrg_stats_judy_mem(MRG *mrg, size_t partition, int64_t judy_m

94 __atomic_add_fetch(&mrg->index[partition].stats.size, judy_mem, __ATOMIC_RELAXED);

95 }

97 -static inline time_t mrg_metric_get_first_time_s_smart(MRG *mrg __maybe_unused, METRIC *metric) {

97 +static ALWAYS_INLINE time_t mrg_metric_get_first_time_s_smart(MRG *mrg __maybe_unused, METRIC *metric) {

98 time_t first_time_s = __atomic_load_n(&metric->first_time_s, __ATOMIC_RELAXED);

100 if(first_time_s <= 0) {

@@ -140,7 +140,7 @@ static inline void metric_log(MRG *mrg __maybe_unused, METRIC *metric, const cha

140 );

141 }

142

143 -static inline bool acquired_metric_has_retention(MRG *mrg, METRIC *metric) {

143 +static ALWAYS_INLINE bool acquired_metric_has_retention(MRG *mrg, METRIC *metric) {

144 time_t first, last;

145 mrg_metric_get_retention(mrg, metric, &first, &last, NULL);

146 bool rc = (first != 0 && last != 0 && first <= last);

@@ -151,7 +151,7 @@ static inline bool acquired_metric_has_retention(MRG *mrg, METRIC *metric) {

151 return rc;

152 }

153

154 -static inline void acquired_for_deletion_metric_delete(MRG *mrg, METRIC *metric) {

154 +static ALWAYS_INLINE void acquired_for_deletion_metric_delete(MRG *mrg, METRIC *metric) {

155 JudyAllocThreadPulseReset();

156

157 size_t partition = metric->partition;

@@ -191,7 +191,7 @@ static inline void acquired_for_deletion_metric_delete(MRG *mrg, METRIC *metric)

191 mrg_stats_judy_mem(mrg, partition, JudyAllocThreadPulseGetAndReset());

192 }

193

194 -static inline bool metric_acquire(MRG *mrg, METRIC *metric) {

194 +static ALWAYS_INLINE bool metric_acquire(MRG *mrg, METRIC *metric) {

195 REFCOUNT rc = refcount_acquire_advanced(&metric->refcount);

196 if(!REFCOUNT_ACQUIRED(rc))

197 return false;

@@ -206,7 +206,7 @@ static inline bool metric_acquire(MRG *mrg, METRIC *metric) {

206 return true;

207 }

208

209 -static inline bool metric_release(MRG *mrg, METRIC *metric) {

209 +static ALWAYS_INLINE bool metric_release(MRG *mrg, METRIC *metric) {

210 size_t partition = metric->partition;

211

212 REFCOUNT refcount = refcount_release(&metric->refcount);

@@ -226,7 +226,7 @@ static inline bool metric_release(MRG *mrg, METRIC *metric) {

226 return refcount == REFCOUNT_DELETED;

227 }

228

229 -static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {

229 +static ALWAYS_INLINE METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {

230 JudyAllocThreadPulseReset();

231

232 UUIDMAP_ID id = uuidmap_create(*entry->uuid);

@@ -299,7 +299,7 @@ static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *r

299 return metric;

300 }

301

302 -static inline METRIC *metric_get_and_acquire_by_id(MRG *mrg, UUIDMAP_ID id, Word_t section) {

302 +static ALWAYS_INLINE METRIC *metric_get_and_acquire_by_id(MRG *mrg, UUIDMAP_ID id, Word_t section) {

303 size_t partition = uuidmap_id_to_partition(id);

304

305 while(1) {

@@ -357,7 +357,7 @@ struct aral_statistics *mrg_aral_stats(void) {

357 return &mrg_aral_statistics;

358 }

359

360 -inline void mrg_destroy(MRG *mrg __maybe_unused) {

360 +ALWAYS_INLINE void mrg_destroy(MRG *mrg __maybe_unused) {

361 // no destruction possible

362 // we can't traverse the metrics list

363

@@ -367,54 +367,54 @@ inline void mrg_destroy(MRG *mrg __maybe_unused) {

367 pulse_aral_unregister(mrg->index[0].aral);

368 }

369

370 -inline METRIC *mrg_metric_add_and_acquire(MRG *mrg, MRG_ENTRY entry, bool *ret) {

370 +ALWAYS_INLINE METRIC *mrg_metric_add_and_acquire(MRG *mrg, MRG_ENTRY entry, bool *ret) {

371 // internal_fatal(entry.latest_time_s > max_acceptable_collected_time(),

372 // "DBENGINE METRIC: metric latest time is in the future");

373

374 return metric_add_and_acquire(mrg, &entry, ret);

375 }

376

377 -inline METRIC *mrg_metric_get_and_acquire_by_uuid(MRG *mrg, nd_uuid_t *uuid, Word_t section) {

377 +ALWAYS_INLINE METRIC *mrg_metric_get_and_acquire_by_uuid(MRG *mrg, nd_uuid_t *uuid, Word_t section) {

378 UUIDMAP_ID id = uuidmap_create(*uuid);

379 METRIC *metric = metric_get_and_acquire_by_id(mrg, id, section);

380 uuidmap_free(id);

381 return metric;

382 }

383

384 -inline METRIC *mrg_metric_get_and_acquire_by_id(MRG *mrg, UUIDMAP_ID id, Word_t section) {

384 +ALWAYS_INLINE METRIC *mrg_metric_get_and_acquire_by_id(MRG *mrg, UUIDMAP_ID id, Word_t section) {

385 return metric_get_and_acquire_by_id(mrg, id, section);

386 }

387

388 -inline bool mrg_metric_release_and_delete(MRG *mrg, METRIC *metric) {

388 +ALWAYS_INLINE bool mrg_metric_release_and_delete(MRG *mrg, METRIC *metric) {

389 return metric_release(mrg, metric);

390 }

391

392 -inline METRIC *mrg_metric_dup(MRG *mrg, METRIC *metric) {

392 +ALWAYS_INLINE METRIC *mrg_metric_dup(MRG *mrg, METRIC *metric) {

393 metric_acquire(mrg, metric);

394 return metric;

395 }

396

397 -inline void mrg_metric_release(MRG *mrg, METRIC *metric) {

397 +ALWAYS_INLINE void mrg_metric_release(MRG *mrg, METRIC *metric) {

398 metric_release(mrg, metric);

399 }

400

401 -inline Word_t mrg_metric_id(MRG *mrg __maybe_unused, METRIC *metric) {

401 +ALWAYS_INLINE Word_t mrg_metric_id(MRG *mrg __maybe_unused, METRIC *metric) {

402 return (Word_t)metric;

403 }

404

405 -inline nd_uuid_t *mrg_metric_uuid(MRG *mrg __maybe_unused, METRIC *metric) {

405 +ALWAYS_INLINE nd_uuid_t *mrg_metric_uuid(MRG *mrg __maybe_unused, METRIC *metric) {

406 return uuidmap_uuid_ptr(metric->uuid);

407 }

408

409 -inline UUIDMAP_ID mrg_metric_uuidmap_id_dup(MRG *mrg __maybe_unused, METRIC *metric) {

409 +ALWAYS_INLINE UUIDMAP_ID mrg_metric_uuidmap_id_dup(MRG *mrg __maybe_unused, METRIC *metric) {

410 return uuidmap_dup(metric->uuid);

411 }

412

413 -inline Word_t mrg_metric_section(MRG *mrg __maybe_unused, METRIC *metric) {

413 +ALWAYS_INLINE Word_t mrg_metric_section(MRG *mrg __maybe_unused, METRIC *metric) {

414 return metric->section;

415 }

416

417 -inline bool mrg_metric_set_first_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s) {

417 +ALWAYS_INLINE bool mrg_metric_set_first_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s) {

418 internal_fatal(first_time_s < 0, "DBENGINE METRIC: timestamp is negative");

419

420 if(first_time_s == LONG_MAX)

@@ -428,7 +428,7 @@ inline bool mrg_metric_set_first_time_s(MRG *mrg __maybe_unused, METRIC *metric,

428 return true;

429 }

430

431 -inline void mrg_metric_expand_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s, time_t last_time_s, uint32_t update_every_s) {

431 +ALWAYS_INLINE void mrg_metric_expand_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s, time_t last_time_s, uint32_t update_every_s) {

432 internal_fatal(first_time_s < 0 || last_time_s < 0,

433 "DBENGINE METRIC: timestamp is negative");

434 internal_fatal(first_time_s > max_acceptable_collected_time(),

@@ -450,16 +450,16 @@ inline void mrg_metric_expand_retention(MRG *mrg __maybe_unused, METRIC *metric,

450 set_metric_field_with_condition(metric->latest_update_every_s, update_every_s, _current <= 0);

451 }

452

453 -inline bool mrg_metric_set_first_time_s_if_bigger(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s) {

453 +ALWAYS_INLINE bool mrg_metric_set_first_time_s_if_bigger(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s) {

454 internal_fatal(first_time_s < 0, "DBENGINE METRIC: timestamp is negative");

455 return set_metric_field_with_condition(metric->first_time_s, first_time_s, _wanted != 0 && _wanted != LONG_MAX && _wanted > _current);

456 }

457

458 -inline time_t mrg_metric_get_first_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

458 +ALWAYS_INLINE time_t mrg_metric_get_first_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

459 return mrg_metric_get_first_time_s_smart(mrg, metric);

460 }

461

462 -inline void mrg_metric_get_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t *first_time_s, time_t *last_time_s, uint32_t *update_every_s) {

462 +ALWAYS_INLINE_HOT void mrg_metric_get_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t *first_time_s, time_t *last_time_s, uint32_t *update_every_s) {

463 time_t clean = __atomic_load_n(&metric->latest_time_s_clean, __ATOMIC_RELAXED);

464 time_t hot = __atomic_load_n(&metric->latest_time_s_hot, __ATOMIC_RELAXED);

465

@@ -469,7 +469,7 @@ inline void mrg_metric_get_retention(MRG *mrg __maybe_unused, METRIC *metric, ti

469 *update_every_s = __atomic_load_n(&metric->latest_update_every_s, __ATOMIC_RELAXED);

470 }

471

472 -inline bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {

472 +ALWAYS_INLINE bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {

473 internal_fatal(latest_time_s < 0, "DBENGINE METRIC: timestamp is negative");

474

475 // internal_fatal(latest_time_s > max_acceptable_collected_time(),

@@ -490,7 +490,7 @@ inline bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *

490 }

491

492 // returns true when metric still has retention

493 -inline bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metric) {

493 +ALWAYS_INLINE bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metric) {

494 Word_t section = mrg_metric_section(mrg, metric);

495 bool do_again = false;

496 size_t countdown = 5;

@@ -538,7 +538,7 @@ inline bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metr

538 return (first && last && first < last);

539 }

540

541 -inline bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {

541 +ALWAYS_INLINE bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {

542 internal_fatal(latest_time_s < 0, "DBENGINE METRIC: timestamp is negative");

543

544 // internal_fatal(latest_time_s > max_acceptable_collected_time(),

@@ -552,38 +552,38 @@ inline bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *me

552 return false;

553 }

554

555 -inline time_t mrg_metric_get_latest_clean_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

555 +ALWAYS_INLINE time_t mrg_metric_get_latest_clean_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

556 time_t clean = __atomic_load_n(&metric->latest_time_s_clean, __ATOMIC_RELAXED);

557 return clean;

558 }

559

560 -inline time_t mrg_metric_get_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

560 +ALWAYS_INLINE time_t mrg_metric_get_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

561 time_t clean = __atomic_load_n(&metric->latest_time_s_clean, __ATOMIC_RELAXED);

562 time_t hot = __atomic_load_n(&metric->latest_time_s_hot, __ATOMIC_RELAXED);

563

564 return MAX(clean, hot);

565 }

566

567 -inline bool mrg_metric_set_update_every(MRG *mrg __maybe_unused, METRIC *metric, uint32_t update_every_s) {

568 - if(update_every_s > 0)

567 +ALWAYS_INLINE bool mrg_metric_set_update_every(MRG *mrg __maybe_unused, METRIC *metric, uint32_t update_every_s) {

568 + if(likely(update_every_s > 0))

569 return set_metric_field_with_condition(metric->latest_update_every_s, update_every_s, true);

570

571 return false;

572 }

573

574 -inline bool mrg_metric_set_update_every_s_if_zero(MRG *mrg __maybe_unused, METRIC *metric, uint32_t update_every_s) {

575 - if(update_every_s > 0)

574 +ALWAYS_INLINE_HOT bool mrg_metric_set_update_every_s_if_zero(MRG *mrg __maybe_unused, METRIC *metric, uint32_t update_every_s) {

575 + if(likely(update_every_s > 0))

576 return set_metric_field_with_condition(metric->latest_update_every_s, update_every_s, _current <= 0);

577

578 return false;

579 }

580

581 -inline uint32_t mrg_metric_get_update_every_s(MRG *mrg __maybe_unused, METRIC *metric) {

581 +ALWAYS_INLINE uint32_t mrg_metric_get_update_every_s(MRG *mrg __maybe_unused, METRIC *metric) {

582 return __atomic_load_n(&metric->latest_update_every_s, __ATOMIC_RELAXED);

583 }

584

585 #ifdef NETDATA_INTERNAL_CHECKS

586 -inline bool mrg_metric_set_writer(MRG *mrg, METRIC *metric) {

586 +ALWAYS_INLINE bool mrg_metric_set_writer(MRG *mrg, METRIC *metric) {

587 pid_t expected = __atomic_load_n(&metric->writer, __ATOMIC_RELAXED);

588 pid_t wanted = gettid_cached();

589 bool done = true;

@@ -603,7 +603,7 @@ inline bool mrg_metric_set_writer(MRG *mrg, METRIC *metric) {

603 return done;

604 }

605

606 -inline bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric) {

606 +ALWAYS_INLINE bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric) {

607 // this function can be called from a different thread than the one than the writer

608

609 pid_t expected = __atomic_load_n(&metric->writer, __ATOMIC_RELAXED);

src/database/engine/metric.h

+1 -1

@@ -19,7 +19,7 @@ struct mrg_statistics {

19 // --- non-atomic --- under a write lock

21 size_t entries;

22 - ssize_t size; // total memory used, with indexing

22 + int64_t size; // total memory used, with indexing

24 size_t additions;

25 size_t additions_duplicate;

src/database/engine/page.c

+16 -13

@@ -442,7 +442,7 @@ ALWAYS_INLINE void dbengine_extent_free(void *extent, size_t size) {

442 // ----------------------------------------------------------------------------

443 // management api

444

445 -PGD *pgd_create(uint8_t type, uint32_t slots) {

445 +ALWAYS_INLINE PGD *pgd_create(uint8_t type, uint32_t slots) {

446

447 PGD *pg = pgd_alloc(true); // this is malloc'd !

448 pg->type = type;

@@ -493,7 +493,7 @@ PGD *pgd_create(uint8_t type, uint32_t slots) {

493 return pg;

494 }

495

496 -PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size) {

496 +ALWAYS_INLINE PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size) {

497

498 if (!size || size < page_type_size[type])

499 return PGD_EMPTY;

@@ -714,7 +714,7 @@ ALWAYS_INLINE uint32_t pgd_capacity(PGD *pg) {

714 }

715

716 // return the overall memory footprint of the page, including all its structures and overheads

717 -uint32_t pgd_memory_footprint(PGD *pg)

717 +ALWAYS_INLINE uint32_t pgd_memory_footprint(PGD *pg)

718 {

719 if (!pg)

720 return 0;

@@ -884,15 +884,17 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size)

884 // data collection

885

886 // returns additional memory that may have been allocated to store this point

887 -ALWAYS_INLINE size_t pgd_append_point(PGD *pg,

888 - usec_t point_in_time_ut __maybe_unused,

889 - NETDATA_DOUBLE n,

890 - NETDATA_DOUBLE min_value,

891 - NETDATA_DOUBLE max_value,

892 - uint16_t count,

893 - uint16_t anomaly_count,

894 - SN_FLAGS flags,

895 - uint32_t expected_slot)

887 +ALWAYS_INLINE_HOT_FLATTEN

888 +size_t pgd_append_point(

889 + PGD *pg,

890 + usec_t point_in_time_ut __maybe_unused,

891 + NETDATA_DOUBLE n,

892 + NETDATA_DOUBLE min_value,

893 + NETDATA_DOUBLE max_value,

894 + uint16_t count,

895 + uint16_t anomaly_count,

896 + SN_FLAGS flags,

897 + uint32_t expected_slot)

898 {

899 if (pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING)

900 pgd_fatal(pg, "Data collection on page already scheduled for flushing");

@@ -1037,7 +1039,8 @@ void pgdc_reset(PGDC *pgdc, PGD *pgd, uint32_t position)

1039 pgdc_seek(pgdc, position);

1040 }

1041

1040 -ALWAYS_INLINE bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused, STORAGE_POINT *sp)

1042 +ALWAYS_INLINE_HOT_FLATTEN

1043 +bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused, STORAGE_POINT *sp)

1044 {

1045 if (!pgdc->pgd || pgdc->pgd == PGD_EMPTY || pgdc->position >= pgdc->slots)

1046 {

src/database/engine/pagecache.c

+19 -19

@@ -81,7 +81,7 @@ static void extent_cache_flush_dirty_page_callback(PGC *cache __maybe_unused, PG

81 ;

82 }

84 -inline TIME_RANGE_COMPARE is_page_in_time_range(time_t page_first_time_s, time_t page_last_time_s, time_t wanted_start_time_s, time_t wanted_end_time_s) {

84 +ALWAYS_INLINE_HOT TIME_RANGE_COMPARE is_page_in_time_range(time_t page_first_time_s, time_t page_last_time_s, time_t wanted_start_time_s, time_t wanted_end_time_s) {

85 // page_first_time_s <= wanted_end_time_s && page_last_time_s >= wanted_start_time_s

87 if(page_last_time_s < wanted_start_time_s)

@@ -93,7 +93,7 @@ inline TIME_RANGE_COMPARE is_page_in_time_range(time_t page_first_time_s, time_t

93 return PAGE_IS_IN_RANGE;

94 }

96 -static inline struct page_details *pdc_find_page_for_time(

96 +static ALWAYS_INLINE_HOT struct page_details *pdc_find_page_for_time(

97 Pcvoid_t PArray,

98 time_t wanted_time_s,

99 size_t *gaps,

@@ -214,7 +214,7 @@ static inline struct page_details *pdc_find_page_for_time(

214 return NULL;

215 }

216

217 -static size_t get_page_list_from_pgc(PGC *cache, METRIC *metric, struct rrdengine_instance *ctx,

217 +static ALWAYS_INLINE_HOT size_t get_page_list_from_pgc(PGC *cache, METRIC *metric, struct rrdengine_instance *ctx,

218 time_t wanted_start_time_s, time_t wanted_end_time_s,

219 Pvoid_t *JudyL_page_array, size_t *cache_gaps,

220 bool open_cache_mode, PDC_PAGE_STATUS tags) {

@@ -357,7 +357,7 @@ static void pgc_inject_gap(struct rrdengine_instance *ctx, METRIC *metric, time_

357 pgc_page_release(main_cache, page);

358 }

359

360 -static size_t list_has_time_gaps(

360 +static ALWAYS_INLINE_HOT size_t list_has_time_gaps(

361 struct rrdengine_instance *ctx,

362 METRIC *metric,

363 Pvoid_t JudyL_page_array,

@@ -491,7 +491,7 @@ static size_t list_has_time_gaps(

491 // ----------------------------------------------------------------------------

492

493 typedef void (*page_found_callback_t)(PGC_PAGE *page, void *data);

494 -static size_t get_page_list_from_journal_v2(struct rrdengine_instance *ctx, METRIC *metric, usec_t start_time_ut, usec_t end_time_ut, page_found_callback_t callback, void *callback_data) {

494 +static ALWAYS_INLINE_HOT size_t get_page_list_from_journal_v2(struct rrdengine_instance *ctx, METRIC *metric, usec_t start_time_ut, usec_t end_time_ut, page_found_callback_t callback, void *callback_data) {

495 nd_uuid_t *uuid = mrg_metric_uuid(main_mrg, metric);

496 Word_t metric_id = mrg_metric_id(main_mrg, metric);

497

@@ -629,7 +629,7 @@ void add_page_details_from_journal_v2(PGC_PAGE *page, void *JudyL_pptr) {

629 // Pvalue of the judy will be the end time for that page

630 // DBENGINE2:

631 #define time_delta(finish, pass) do { if(pass) { usec_t t = pass; (pass) = (finish) - (pass); (finish) = t; } } while(0)

632 -static Pvoid_t get_page_list(

632 +static ALWAYS_INLINE_HOT Pvoid_t get_page_list(

633 struct rrdengine_instance *ctx,

634 METRIC *metric,

635 usec_t start_time_ut,

@@ -756,7 +756,7 @@ we_are_done:

756 return JudyL_page_array;

757 }

758

759 -inline void rrdeng_prep_wait(PDC *pdc) {

759 +ALWAYS_INLINE void rrdeng_prep_wait(PDC *pdc) {

760 if (unlikely(pdc && !pdc->prep_done)) {

761 usec_t started_ut = now_monotonic_usec();

762 completion_wait_for(&pdc->prep_completion);

@@ -765,7 +765,7 @@ inline void rrdeng_prep_wait(PDC *pdc) {

765 }

766 }

767

768 -void rrdeng_prep_query(struct page_details_control *pdc, bool worker) {

768 +ALWAYS_INLINE_HOT void rrdeng_prep_query(struct page_details_control *pdc, bool worker) {

769 if(worker)

770 worker_is_busy(UV_EVENT_DBENGINE_QUERY);

771

@@ -820,7 +820,7 @@ void rrdeng_prep_query(struct page_details_control *pdc, bool worker) {

820 * @param end_time_ut inclusive ending time in usec

821 * @return 1 / 0 (pages found or not found)

822 */

823 -void pg_cache_preload(struct rrdeng_query_handle *handle) {

823 +ALWAYS_INLINE_HOT void pg_cache_preload(struct rrdeng_query_handle *handle) {

824 if (unlikely(!handle || !handle->metric))

825 return;

826

@@ -1045,31 +1045,31 @@ void pgc_open_add_hot_page(Word_t section, Word_t metric_id, time_t start_time_s

1045 pgc_page_release(open_cache, (PGC_PAGE *)page);

1046 }

1047

1048 -size_t dynamic_open_cache_size(void) {

1049 - size_t main_wanted_cache_size = pgc_get_wanted_cache_size(main_cache);

1050 - size_t target_size = main_wanted_cache_size / 100 * 5;

1048 +int64_t dynamic_open_cache_size(void) {

1049 + int64_t main_wanted_cache_size = pgc_get_wanted_cache_size(main_cache);

1050 + int64_t target_size = main_wanted_cache_size / 100 * 5;

1051

1052 if(target_size < 2 * 1024 * 1024)

1053 target_size = 2 * 1024 * 1024;

1054

1055 - size_t main_current_cache_size = pgc_get_current_cache_size(main_cache);

1055 + int64_t main_current_cache_size = pgc_get_current_cache_size(main_cache);

1056

1057 - size_t main_free_cache_size = (main_wanted_cache_size > main_current_cache_size) ?

1057 + int64_t main_free_cache_size = (main_wanted_cache_size > main_current_cache_size) ?

1058 main_wanted_cache_size - main_current_cache_size : 0;

1059

1060 return target_size + main_free_cache_size;

1061 }

1062

1063 -size_t dynamic_extent_cache_size(void) {

1064 - size_t main_wanted_cache_size = pgc_get_wanted_cache_size(main_cache);

1065 - size_t target_size = main_wanted_cache_size / 100 * 30;

1063 +int64_t dynamic_extent_cache_size(void) {

1064 + int64_t main_wanted_cache_size = pgc_get_wanted_cache_size(main_cache);

1065 + int64_t target_size = main_wanted_cache_size / 100 * 30;

1066

1067 if(target_size < 5 * 1024 * 1024)

1068 target_size = 5 * 1024 * 1024;

1069

1070 - size_t main_current_cache_size = pgc_get_current_cache_size(main_cache);

1070 + int64_t main_current_cache_size = pgc_get_current_cache_size(main_cache);

1071

1072 - size_t main_free_cache_size = (main_wanted_cache_size > main_current_cache_size) ?

1072 + int64_t main_free_cache_size = (main_wanted_cache_size > main_current_cache_size) ?

1073 main_wanted_cache_size - main_current_cache_size : 0;

1074

1075 return target_size + main_free_cache_size;

src/database/engine/pdc.c

+26 -26

@@ -61,13 +61,13 @@ void pdc_init(void) {

61 pulse_aral_register(pdc_globals.pdc.ar, "pdc");

62 }

64 -PDC *pdc_get(void) {

64 +ALWAYS_INLINE PDC *pdc_get(void) {

65 PDC *pdc = aral_mallocz(pdc_globals.pdc.ar);

66 memset(pdc, 0, sizeof(PDC));

67 return pdc;

68 }

70 -static void pdc_release(PDC *pdc) {

70 +static ALWAYS_INLINE void pdc_release(PDC *pdc) {

71 aral_freez(pdc_globals.pdc.ar, pdc);

72 }

@@ -90,13 +90,13 @@ void page_details_init(void) {

90 pulse_aral_register(pdc_globals.pd.ar, "pd");

91 }

93 -struct page_details *page_details_get(void) {

93 +ALWAYS_INLINE struct page_details *page_details_get(void) {

94 struct page_details *pd = aral_mallocz(pdc_globals.pd.ar);

95 memset(pd, 0, sizeof(struct page_details));

96 return pd;

97 }

99 -static void page_details_release(struct page_details *pd) {

99 +static ALWAYS_INLINE void page_details_release(struct page_details *pd) {

100 aral_freez(pdc_globals.pd.ar, pd);

101 }

102

@@ -119,13 +119,13 @@ void epdl_init(void) {

119 pulse_aral_register(pdc_globals.epdl.ar, "epdl");

120 }

121

122 -static EPDL *epdl_get(void) {

122 +static ALWAYS_INLINE EPDL *epdl_get(void) {

123 EPDL *epdl = aral_mallocz(pdc_globals.epdl.ar);

124 memset(epdl, 0, sizeof(EPDL));

125 return epdl;

126 }

127

128 -static void epdl_release(EPDL *epdl) {

128 +static ALWAYS_INLINE void epdl_release(EPDL *epdl) {

129 aral_freez(pdc_globals.epdl.ar, epdl);

130 }

131

@@ -149,13 +149,13 @@ void deol_init(void) {

149 pulse_aral_register(pdc_globals.deol.ar, "deol");

150 }

151

152 -static DEOL *deol_get(void) {

152 +static ALWAYS_INLINE DEOL *deol_get(void) {

153 DEOL *deol = aral_mallocz(pdc_globals.deol.ar);

154 memset(deol, 0, sizeof(DEOL));

155 return deol;

156 }

157

158 -static void deol_release(DEOL *deol) {

158 +static ALWAYS_INLINE void deol_release(DEOL *deol) {

159 aral_freez(pdc_globals.deol.ar, deol);

160 }

161

@@ -224,7 +224,7 @@ void extent_buffer_cleanup1(void) {

224 }

225 }

226

227 -struct extent_buffer *extent_buffer_get(size_t size) {

227 +ALWAYS_INLINE struct extent_buffer *extent_buffer_get(size_t size) {

228 internal_fatal(size > extent_buffer_globals.max_size, "DBENGINE: extent size is too big");

229

230 struct extent_buffer *eb = NULL;

@@ -259,7 +259,7 @@ struct extent_buffer *extent_buffer_get(size_t size) {

259 return eb;

260 }

261

262 -void extent_buffer_release(struct extent_buffer *eb) {

262 +ALWAYS_INLINE void extent_buffer_release(struct extent_buffer *eb) {

263 if(unlikely(!eb)) return;

264

265 spinlock_lock(&extent_buffer_globals.protected.spinlock);

@@ -275,7 +275,7 @@ size_t extent_buffer_cache_size(void) {

275 // ----------------------------------------------------------------------------

276 // epdl logic

277

278 -static void epdl_destroy(EPDL *epdl)

278 +static ALWAYS_INLINE void epdl_destroy(EPDL *epdl)

279 {

280 Pvoid_t *pd_by_start_time_s_JudyL;

281 Word_t metric_id_index = 0;

@@ -289,7 +289,7 @@ static void epdl_destroy(EPDL *epdl)

289 epdl_release(epdl);

290 }

291

292 -static void epdl_mark_all_not_loaded_pages_as_failed(EPDL *epdl, PDC_PAGE_STATUS tags, size_t *statistics_counter)

292 +static ALWAYS_INLINE void epdl_mark_all_not_loaded_pages_as_failed(EPDL *epdl, PDC_PAGE_STATUS tags, size_t *statistics_counter)

293 {

294 size_t pages_matched = 0;

295

@@ -355,7 +355,7 @@ static bool epdl_check_if_pages_are_already_in_cache(struct rrdengine_instance *

355 // ----------------------------------------------------------------------------

356 // PDC logic

357

358 -static void pdc_destroy(PDC *pdc) {

358 +static ALWAYS_INLINE void pdc_destroy(PDC *pdc) {

359 mrg_metric_release(main_mrg, pdc->metric);

360 completion_destroy(&pdc->prep_completion);

361 completion_destroy(&pdc->page_completion);

@@ -406,7 +406,7 @@ static void pdc_destroy(PDC *pdc) {

406 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_load_fail_cancelled, cancelled, __ATOMIC_RELAXED);

407 }

408

409 -void pdc_acquire(PDC *pdc) {

409 +ALWAYS_INLINE void pdc_acquire(PDC *pdc) {

410 spinlock_lock(&pdc->refcount_spinlock);

411

412 if(pdc->refcount < 1)

@@ -416,7 +416,7 @@ void pdc_acquire(PDC *pdc) {

416 spinlock_unlock(&pdc->refcount_spinlock);

417 }

418

419 -bool pdc_release_and_destroy_if_unreferenced(PDC *pdc, bool worker, bool router __maybe_unused) {

419 +ALWAYS_INLINE bool pdc_release_and_destroy_if_unreferenced(PDC *pdc, bool worker, bool router __maybe_unused) {

420 if(unlikely(!pdc))

421 return true;

422

@@ -445,22 +445,22 @@ bool pdc_release_and_destroy_if_unreferenced(PDC *pdc, bool worker, bool router

445 return false;

446 }

447

448 -void epdl_cmd_queued(void *epdl_ptr, struct rrdeng_cmd *cmd) {

448 +ALWAYS_INLINE void epdl_cmd_queued(void *epdl_ptr, struct rrdeng_cmd *cmd) {

449 EPDL *epdl = epdl_ptr;

450 epdl->cmd = cmd;

451 }

452

453 -void epdl_cmd_dequeued(void *epdl_ptr) {

453 +ALWAYS_INLINE void epdl_cmd_dequeued(void *epdl_ptr) {

454 EPDL *epdl = epdl_ptr;

455 epdl->cmd = NULL;

456 }

457

458 -static inline struct rrdeng_cmd *epdl_get_cmd(void *epdl_ptr) {

458 +static ALWAYS_INLINE struct rrdeng_cmd *epdl_get_cmd(void *epdl_ptr) {

459 EPDL *epdl = epdl_ptr;

460 return epdl->cmd;

461 }

462

463 -static EPDL_EXTENT *epdl_find_extent_base(EPDL *epdl) {

463 +static ALWAYS_INLINE EPDL_EXTENT *epdl_find_extent_base(EPDL *epdl) {

464 EPDL_EXTENT *e = NULL;

465 rw_spinlock_read_lock(&epdl->datafile->extent_epdl.spinlock);

466 Pvoid_t *PValue = JudyLGet(epdl->datafile->extent_epdl.epdl_per_extent, epdl->extent_offset, PJE0);

@@ -492,7 +492,7 @@ static EPDL_EXTENT *epdl_find_extent_base(EPDL *epdl) {

492 return e;

493 }

494

495 -static bool epdl_pending_add(EPDL *epdl) {

495 +static ALWAYS_INLINE bool epdl_pending_add(EPDL *epdl) {

496 EPDL_EXTENT *e = epdl_find_extent_base(epdl);

497 spinlock_lock(&e->spinlock);

498

@@ -519,14 +519,14 @@ static bool epdl_pending_add(EPDL *epdl) {

519 return added_new;

520 }

521

522 -static void epdl_pending_del(EPDL *epdl) {

522 +static ALWAYS_INLINE void epdl_pending_del(EPDL *epdl) {

523 EPDL_EXTENT *e = epdl_find_extent_base(epdl);

524 spinlock_lock(&e->spinlock);

525 e->base = NULL;

526 spinlock_unlock(&e->spinlock);

527 }

528

529 -void pdc_to_epdl_router(struct rrdengine_instance *ctx, PDC *pdc, execute_extent_page_details_list_t exec_first_extent_list, execute_extent_page_details_list_t exec_rest_extent_list)

529 +ALWAYS_INLINE_HOT void pdc_to_epdl_router(struct rrdengine_instance *ctx, PDC *pdc, execute_extent_page_details_list_t exec_first_extent_list, execute_extent_page_details_list_t exec_rest_extent_list)

530 {

531 Pvoid_t *PValue;

532 Pvoid_t *PValue1;

@@ -660,7 +660,7 @@ void collect_page_flags_to_buffer(BUFFER *wb, RRDENG_COLLECT_PAGE_FLAGS flags) {

660 buffer_strcat(wb, "STEP_UNALIGNED");

661 }

662

663 -inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_extent_page_descr *descr, time_t now_s, uint32_t overwrite_zero_update_every_s, bool have_read_error) {

663 +ALWAYS_INLINE VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_extent_page_descr *descr, time_t now_s, uint32_t overwrite_zero_update_every_s, bool have_read_error) {

664 time_t start_time_s = (time_t) (descr->start_time_ut / USEC_PER_SEC);

665

666 time_t end_time_s = 0;

@@ -695,7 +695,7 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_

695 "loaded", 0);

696 }

697

698 -VALIDATED_PAGE_DESCRIPTOR validate_page(

698 +ALWAYS_INLINE VALIDATED_PAGE_DESCRIPTOR validate_page(

699 nd_uuid_t *uuid,

700 time_t start_time_s,

701 time_t end_time_s,

@@ -862,7 +862,7 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(

862 return vd;

863 }

864

865 -static inline struct page_details *epdl_get_pd_load_link_list_from_metric_start_time(EPDL *epdl, Word_t metric_id, time_t start_time_s) {

865 +static ALWAYS_INLINE struct page_details *epdl_get_pd_load_link_list_from_metric_start_time(EPDL *epdl, Word_t metric_id, time_t start_time_s) {

866

867 if(unlikely(epdl->head_to_datafile_extent_queries_pending_for_extent))

868 // stop appending more pages to this epdl

@@ -1239,7 +1239,7 @@ static inline void datafile_extent_read_free(void *buffer) {

1239 posix_memfree(buffer);

1240 }

1241

1242 -void epdl_find_extent_and_populate_pages(struct rrdengine_instance *ctx, EPDL *epdl, bool worker) {

1242 +NOT_INLINE_HOT void epdl_find_extent_and_populate_pages(struct rrdengine_instance *ctx, EPDL *epdl, bool worker) {

1243 if(worker)

1244 worker_is_busy(UV_EVENT_DBENGINE_EXTENT_CACHE_LOOKUP);

1245

src/database/engine/rrdengine.c

+11 -11

@@ -318,13 +318,13 @@ void rrdeng_query_handle_init(void) {

318 pulse_aral_register(rrdeng_main.handles.ar, "query handles");

319 }

320

321 -struct rrdeng_query_handle *rrdeng_query_handle_get(void) {

321 +ALWAYS_INLINE struct rrdeng_query_handle *rrdeng_query_handle_get(void) {

322 struct rrdeng_query_handle *handle = aral_mallocz(rrdeng_main.handles.ar);

323 memset(handle, 0, sizeof(struct rrdeng_query_handle));

324 return handle;

325 }

326

327 -void rrdeng_query_handle_release(struct rrdeng_query_handle *handle) {

327 +ALWAYS_INLINE void rrdeng_query_handle_release(struct rrdeng_query_handle *handle) {

328 aral_freez(rrdeng_main.handles.ar, handle);

329 }

330

@@ -456,15 +456,15 @@ static inline STORAGE_PRIORITY rrdeng_enq_cmd_map_opcode_to_priority(enum rrdeng

456 return priority;

457 }

458

459 -void rrdeng_enqueue_epdl_cmd(struct rrdeng_cmd *cmd) {

459 +ALWAYS_INLINE void rrdeng_enqueue_epdl_cmd(struct rrdeng_cmd *cmd) {

460 epdl_cmd_queued(cmd->data, cmd);

461 }

462

463 -void rrdeng_dequeue_epdl_cmd(struct rrdeng_cmd *cmd) {

463 +ALWAYS_INLINE void rrdeng_dequeue_epdl_cmd(struct rrdeng_cmd *cmd) {

464 epdl_cmd_dequeued(cmd->data);

465 }

466

467 -void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY priority) {

467 +ALWAYS_INLINE void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY priority) {

468 spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);

469

470 struct rrdeng_cmd *cmd = get_cmd_cb(data);

@@ -481,7 +481,7 @@ void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY

481 spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);

482 }

483

484 -void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, void *data, struct completion *completion,

484 +ALWAYS_INLINE void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, void *data, struct completion *completion,

485 enum storage_priority priority, enqueue_callback_t enqueue_cb, dequeue_callback_t dequeue_cb) {

486

487 priority = rrdeng_enq_cmd_map_opcode_to_priority(opcode, priority);

@@ -1491,24 +1491,24 @@ static void *extent_read_tp_worker(struct rrdengine_instance *ctx __maybe_unused

1491 return data;

1492 }

1493

1494 -static void epdl_populate_pages_asynchronously(struct rrdengine_instance *ctx, EPDL *epdl, STORAGE_PRIORITY priority) {

1494 +static NOT_INLINE_HOT void epdl_populate_pages_asynchronously(struct rrdengine_instance *ctx, EPDL *epdl, STORAGE_PRIORITY priority) {

1495 rrdeng_enq_cmd(ctx, RRDENG_OPCODE_EXTENT_READ, epdl, NULL, priority,

1496 rrdeng_enqueue_epdl_cmd, rrdeng_dequeue_epdl_cmd);

1497 }

1498

1499 -void pdc_route_asynchronously(struct rrdengine_instance *ctx, struct page_details_control *pdc) {

1499 +NOT_INLINE_HOT void pdc_route_asynchronously(struct rrdengine_instance *ctx, struct page_details_control *pdc) {

1500 pdc_to_epdl_router(ctx, pdc, epdl_populate_pages_asynchronously, epdl_populate_pages_asynchronously);

1501 }

1502

1503 -void epdl_populate_pages_synchronously(struct rrdengine_instance *ctx, EPDL *epdl, enum storage_priority priority __maybe_unused) {

1503 +NOT_INLINE_HOT void epdl_populate_pages_synchronously(struct rrdengine_instance *ctx, EPDL *epdl, enum storage_priority priority __maybe_unused) {

1504 epdl_find_extent_and_populate_pages(ctx, epdl, false);

1505 }

1506

1507 -void pdc_route_synchronously(struct rrdengine_instance *ctx, struct page_details_control *pdc) {

1507 +NOT_INLINE_HOT void pdc_route_synchronously(struct rrdengine_instance *ctx, struct page_details_control *pdc) {

1508 pdc_to_epdl_router(ctx, pdc, epdl_populate_pages_synchronously, epdl_populate_pages_synchronously);

1509 }

1510

1511 -void pdc_route_synchronously_first(struct rrdengine_instance *ctx, struct page_details_control *pdc) {

1511 +NOT_INLINE_HOT void pdc_route_synchronously_first(struct rrdengine_instance *ctx, struct page_details_control *pdc) {

1512 pdc_to_epdl_router(ctx, pdc, epdl_populate_pages_synchronously, epdl_populate_pages_asynchronously);

1513 }

1514

src/database/engine/rrdengineapi.c

+25 -23

@@ -487,7 +487,7 @@ static PGD *rrdeng_alloc_new_page_data(struct rrdeng_collect_handle *handle, use

487 return d;

488 }

489

490 -static ALWAYS_INLINE void rrdeng_store_metric_append_point(STORAGE_COLLECT_HANDLE *sch,

490 +static ALWAYS_INLINE_HOT void rrdeng_store_metric_append_point(STORAGE_COLLECT_HANDLE *sch,

491 const usec_t point_in_time_ut,

492 const NETDATA_DOUBLE n,

493 const NETDATA_DOUBLE min_value,

@@ -570,14 +570,15 @@ static void store_metric_next_error_log(struct rrdeng_collect_handle *handle __m

570 #endif

571 }

572

573 -ALWAYS_INLINE void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *sch,

574 - const usec_t point_in_time_ut,

575 - const NETDATA_DOUBLE n,

576 - const NETDATA_DOUBLE min_value,

577 - const NETDATA_DOUBLE max_value,

578 - const uint16_t count,

579 - const uint16_t anomaly_count,

580 - const SN_FLAGS flags)

573 +ALWAYS_INLINE_HOT void rrdeng_store_metric_next(

574 + STORAGE_COLLECT_HANDLE *sch,

575 + const usec_t point_in_time_ut,

576 + const NETDATA_DOUBLE n,

577 + const NETDATA_DOUBLE min_value,

578 + const NETDATA_DOUBLE max_value,

579 + const uint16_t count,

580 + const uint16_t anomaly_count,

581 + const SN_FLAGS flags)

582 {

583 timing_step(TIMING_STEP_RRDSET_STORE_METRIC);

584

@@ -708,7 +709,7 @@ void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch

709 #ifdef NETDATA_INTERNAL_CHECKS

710 SPINLOCK global_query_handle_spinlock = SPINLOCK_INITIALIZER;

711 static struct rrdeng_query_handle *global_query_handle_ll = NULL;

711 -static void register_query_handle(struct rrdeng_query_handle *handle) {

712 +static ALWAYS_INLINE void register_query_handle(struct rrdeng_query_handle *handle) {

713 handle->query_pid = gettid_cached();

714 handle->started_time_s = now_realtime_sec();

715

@@ -716,7 +717,7 @@ static void register_query_handle(struct rrdeng_query_handle *handle) {

717 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(global_query_handle_ll, handle, prev, next);

718 spinlock_unlock(&global_query_handle_spinlock);

719 }

719 -static void unregister_query_handle(struct rrdeng_query_handle *handle) {

720 +static ALWAYS_INLINE void unregister_query_handle(struct rrdeng_query_handle *handle) {

721 spinlock_lock(&global_query_handle_spinlock);

722 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(global_query_handle_ll, handle, prev, next);

723 spinlock_unlock(&global_query_handle_spinlock);

@@ -734,11 +735,12 @@ static void unregister_query_handle(struct rrdeng_query_handle *handle __maybe_u

735 * Gets a handle for loading metrics from the database.

736 * The handle must be released with rrdeng_load_metric_final().

737 */

737 -void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *smh,

738 - struct storage_engine_query_handle *seqh,

739 - time_t start_time_s,

740 - time_t end_time_s,

741 - STORAGE_PRIORITY priority)

738 +ALWAYS_INLINE_HOT void rrdeng_load_metric_init(

739 + STORAGE_METRIC_HANDLE *smh,

740 + struct storage_engine_query_handle *seqh,

741 + time_t start_time_s,

742 + time_t end_time_s,

743 + STORAGE_PRIORITY priority)

744 {

745 usec_t started_ut = now_monotonic_usec();

746

@@ -803,7 +805,7 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *smh,

805 }

806 }

807

806 -static ALWAYS_INLINE bool rrdeng_load_page_next(struct storage_engine_query_handle *seqh, bool debug_this __maybe_unused) {

808 +static ALWAYS_INLINE_HOT bool rrdeng_load_page_next(struct storage_engine_query_handle *seqh, bool debug_this __maybe_unused) {

809 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)seqh->handle;

810 struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);

811

@@ -874,7 +876,7 @@ static ALWAYS_INLINE bool rrdeng_load_page_next(struct storage_engine_query_hand

876 // Returns the metric and sets its timestamp into current_time

877 // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)

878 // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES

877 -ALWAYS_INLINE STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh) {

879 +ALWAYS_INLINE_HOT STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh) {

880 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)seqh->handle;

881 STORAGE_POINT sp;

882

@@ -908,7 +910,7 @@ prepare_for_next_iteration:

910 return sp;

911 }

912

911 -int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh) {

913 +ALWAYS_INLINE int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh) {

914 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)seqh->handle;

915 return (handle->now_s > seqh->end_time_s);

916 }

@@ -916,7 +918,7 @@ int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh) {

918 /*

919 * Releases the database reference from the handle for loading metrics.

920 */

919 -void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh)

921 +ALWAYS_INLINE void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh)

922 {

923 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)seqh->handle;

924

@@ -935,7 +937,7 @@ void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh)

937 seqh->handle = NULL;

938 }

939

938 -time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh) {

940 +ALWAYS_INLINE time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh) {

941 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)seqh->handle;

942

943 if(handle->pdc) {

@@ -947,7 +949,7 @@ time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *s

949 return seqh->end_time_s;

950 }

951

950 -time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh) {

952 +ALWAYS_INLINE time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh) {

953 METRIC *metric = (METRIC *)smh;

954 time_t latest_time_s = 0;

955

@@ -957,7 +959,7 @@ time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh) {

959 return latest_time_s;

960 }

961

960 -time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh) {

962 +ALWAYS_INLINE time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh) {

963 METRIC *metric = (METRIC *)smh;

964

965 time_t oldest_time_s = 0;

src/database/engine/rrdengineapi.h

+1 -1

@@ -136,7 +136,7 @@ struct time_and_count {

136 usec_t usec;

137 };

138

139 -static inline void time_and_count_add(struct time_and_count *tc, usec_t dt) {

139 +static ALWAYS_INLINE void time_and_count_add(struct time_and_count *tc, usec_t dt) {

140 __atomic_add_fetch(&tc->count, 1, __ATOMIC_RELAXED);

141 __atomic_add_fetch(&tc->usec, dt, __ATOMIC_RELAXED);

142 }

src/database/rrddim-backfill.c

+1 -1

@@ -6,7 +6,7 @@

6 // ----------------------------------------------------------------------------

7 // fill the gap of a tier

9 -bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {

9 +NOT_INLINE_HOT bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {

10 if(unlikely(tier >= nd_profile.storage_tiers)) return false;

11 #ifdef ENABLE_DBENGINE

12 if(default_backfill == RRD_BACKFILL_NONE) return false;

src/database/rrddim-collection.c

+1 -1

@@ -11,7 +11,7 @@ static inline time_t tier_next_point_time_s(RRDDIM *rd, struct rrddim_tier *t, t

11 return now_s + loop - ((now_s + loop) % loop);

12 }

14 -void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {

14 +ALWAYS_INLINE_HOT void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {

15 if (unlikely(!t->next_point_end_time_s))

16 t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);

src/database/rrdhost-status.c

+1 -1

@@ -258,7 +258,7 @@ static void rrdhost_status_stream_internal(RRDHOST_STATUS *s) {

258 else

259 s->stream.status = RRDHOST_STREAM_STATUS_ONLINE;

260

261 - s->stream.compression = host->sender->compressor.initialized;

261 + s->stream.compression = host->sender->thread.compressor.initialized;

262 }

263 else {

264 s->stream.status = RRDHOST_STREAM_STATUS_OFFLINE;

src/database/storage-engine.h

+77 -14

@@ -49,14 +49,16 @@ typedef struct storage_instance STORAGE_INSTANCE;

49 typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;

50 typedef struct storage_alignment STORAGE_METRICS_GROUP;

52 -// ----------------------------------------------------------------------------

52 +// --------------------------------------------------------------------------------------------------------------------

53 // engine-specific iterator state for dimension data collection

54 +

55 typedef struct storage_collect_handle {

56 STORAGE_ENGINE_BACKEND seb;

57 } STORAGE_COLLECT_HANDLE;

58 -// ------------------------------------------------------------------------

59 +// --------------------------------------------------------------------------------------------------------------------

60 // function pointers for all APIs provided by a storage engine

61 +

62 typedef struct storage_engine_api {

63 // metric management

64 STORAGE_METRIC_HANDLE *(*metric_get_by_id)(STORAGE_INSTANCE *si, UUIDMAP_ID id);

@@ -82,7 +84,7 @@ STORAGE_ENGINE* storage_engine_find(const char* name);

84 STORAGE_ENGINE* storage_engine_foreach_init();

85 STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);

85 -// ----------------------------------------------------------------------------

87 +// --------------------------------------------------------------------------------------------------------------------

88 // Storage tier data for every dimension

90 struct rrddim_tier {

@@ -95,15 +97,16 @@ struct rrddim_tier {

97 STORAGE_COLLECT_HANDLE *sch; // the data collection handle

98 };

98 -// ------------------------------------------------------------------------

100 +// --------------------------------------------------------------------------------------------------------------------

101

102 #include "daemon/config/netdata-conf-db.h"

103

102 -// ------------------------------------------------------------------------

104 +// --------------------------------------------------------------------------------------------------------------------

105 // DATA COLLECTION STORAGE OPS

106

107 STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);

108 STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid);

109 +

110 static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, nd_uuid_t *uuid) {

111 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");

112

@@ -114,8 +117,11 @@ static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_EN

117 return rrddim_metrics_group_get(si, uuid);

118 }

119

120 +// --------------------------------------------------------------------------------------------------------------------

121 +

122 void rrdeng_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);

123 void rrddim_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg);

124 +

125 static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg) {

126 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");

127

@@ -127,8 +133,11 @@ static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND s

133 rrddim_metrics_group_release(si, smg);

134 }

135

136 +// --------------------------------------------------------------------------------------------------------------------

137 +

138 STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);

139 STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg);

140 +

141 static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg) {

142 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");

143

@@ -139,6 +148,8 @@ static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_B

148 return rrddim_collect_init(smh, update_every, smg);

149 }

150

151 +// --------------------------------------------------------------------------------------------------------------------

152 +

153 void rrdeng_store_metric_next(

154 STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,

155 NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,

@@ -149,7 +160,8 @@ void rrddim_collect_store_metric(

160 NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,

161 uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);

162

152 -static inline void storage_engine_store_metric(

163 +ALWAYS_INLINE_HOT_FLATTEN

164 +static void storage_engine_store_metric(

165 STORAGE_COLLECT_HANDLE *sch, usec_t point_in_time_ut,

166 NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,

167 uint16_t count, uint16_t anomaly_count, SN_FLAGS flags) {

@@ -166,7 +178,10 @@ static inline void storage_engine_store_metric(

178 count, anomaly_count, flags);

179 }

180

181 +// --------------------------------------------------------------------------------------------------------------------

182 +

183 uint64_t rrdeng_disk_space_max(STORAGE_INSTANCE *si);

184 +

185 static inline uint64_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {

186 #ifdef ENABLE_DBENGINE

187 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))

@@ -176,7 +191,10 @@ static inline uint64_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND seb

191 return 0;

192 }

193

194 +// --------------------------------------------------------------------------------------------------------------------

195 +

196 uint64_t rrdeng_disk_space_used(STORAGE_INSTANCE *si);

197 +

198 static inline uint64_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {

199 #ifdef ENABLE_DBENGINE

200 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))

@@ -187,7 +205,10 @@ static inline uint64_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND seb

205 return 0;

206 }

207

208 +// --------------------------------------------------------------------------------------------------------------------

209 +

210 uint64_t rrdeng_metrics(STORAGE_INSTANCE *si);

211 +

212 static inline uint64_t storage_engine_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {

213 #ifdef ENABLE_DBENGINE

214 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))

@@ -198,7 +219,10 @@ static inline uint64_t storage_engine_metrics(STORAGE_ENGINE_BACKEND seb __maybe

219 return 0;

220 }

221

222 +// --------------------------------------------------------------------------------------------------------------------

223 +

224 uint64_t rrdeng_samples(STORAGE_INSTANCE *si);

225 +

226 static inline uint64_t storage_engine_samples(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {

227 #ifdef ENABLE_DBENGINE

228 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))

@@ -207,8 +231,10 @@ static inline uint64_t storage_engine_samples(STORAGE_ENGINE_BACKEND seb __maybe

231 return 0;

232 }

233

234 +// --------------------------------------------------------------------------------------------------------------------

235

236 time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *si);

237 +

238 static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {

239 #ifdef ENABLE_DBENGINE

240 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))

@@ -218,7 +244,10 @@ static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND s

244 return now_realtime_sec() - (time_t)(default_rrd_history_entries * nd_profile.update_every);

245 }

246

247 +// --------------------------------------------------------------------------------------------------------------------

248 +

249 size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *si);

250 +

251 static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si __maybe_unused) {

252 #ifdef ENABLE_DBENGINE

253 if(likely(seb == STORAGE_ENGINE_BACKEND_DBENGINE))

@@ -229,8 +258,11 @@ static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND seb

258 return 0;

259 }

260

261 +// --------------------------------------------------------------------------------------------------------------------

262 +

263 void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *sch);

264 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *sch);

265 +

266 static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *sch) {

267 if(unlikely(!sch))

268 return;

@@ -245,10 +277,13 @@ static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *sch) {

277 rrddim_store_metric_flush(sch);

278 }

279

280 +// --------------------------------------------------------------------------------------------------------------------

281 +

282 int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *sch);

283 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *sch);

284 // a finalization function to run after collection is over

285 // returns 1 if it's safe to delete the dimension

286 +

287 static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *sch) {

288 internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");

289

@@ -260,8 +295,11 @@ static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *sch) {

295 return rrddim_collect_finalize(sch);

296 }

297

298 +// --------------------------------------------------------------------------------------------------------------------

299 +

300 void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);

301 void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every);

302 +

303 static inline void storage_engine_store_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch, int update_every) {

304 internal_fatal(!is_valid_backend(sch->seb), "STORAGE: invalid backend");

305

@@ -273,12 +311,14 @@ static inline void storage_engine_store_change_collection_frequency(STORAGE_COLL

311 rrddim_store_metric_change_collection_frequency(sch, update_every);

312 }

313

276 -// ----------------------------------------------------------------------------

314 +// --------------------------------------------------------------------------------------------------------------------

315 // STORAGE ENGINE QUERY OPS

316

317 time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh);

318 time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *smh);

281 -static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {

319 +

320 +ALWAYS_INLINE_HOT_FLATTEN

321 +static time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {

322 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");

323

324 #ifdef ENABLE_DBENGINE

@@ -288,9 +328,13 @@ static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND seb __

328 return rrddim_query_oldest_time_s(smh);

329 }

330

331 +// --------------------------------------------------------------------------------------------------------------------

332 +

333 time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *smh);

334 time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *smh);

293 -static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {

335 +

336 +ALWAYS_INLINE_HOT_FLATTEN

337 +static time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_METRIC_HANDLE *smh) {

338 internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend");

339

340 #ifdef ENABLE_DBENGINE

@@ -300,6 +344,8 @@ static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND seb __m

344 return rrddim_query_latest_time_s(smh);

345 }

346

347 +// --------------------------------------------------------------------------------------------------------------------

348 +

349 void rrdeng_load_metric_init(

350 STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,

351 time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);

@@ -308,7 +354,8 @@ void rrddim_query_init(

354 STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,

355 time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);

356

311 -static inline void storage_engine_query_init(

357 +ALWAYS_INLINE_HOT_FLATTEN

358 +static void storage_engine_query_init(

359 STORAGE_ENGINE_BACKEND seb __maybe_unused,

360 STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_handle *seqh,

361 time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority) {

@@ -322,9 +369,13 @@ static inline void storage_engine_query_init(

369 rrddim_query_init(smh, seqh, start_time_s, end_time_s, priority);

370 }

371

372 +// --------------------------------------------------------------------------------------------------------------------

373 +

374 STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh);

375 STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh);

327 -static ALWAYS_INLINE STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *seqh) {

376 +

377 +ALWAYS_INLINE_HOT_FLATTEN

378 +static STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *seqh) {

379 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");

380

381 #ifdef ENABLE_DBENGINE

@@ -334,9 +385,13 @@ static ALWAYS_INLINE STORAGE_POINT storage_engine_query_next_metric(struct stora

385 return rrddim_query_next_metric(seqh);

386 }

387

388 +// --------------------------------------------------------------------------------------------------------------------

389 +

390 int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh);

391 int rrddim_query_is_finished(struct storage_engine_query_handle *seqh);

339 -static ALWAYS_INLINE int storage_engine_query_is_finished(struct storage_engine_query_handle *seqh) {

392 +

393 +ALWAYS_INLINE_HOT_FLATTEN

394 +static int storage_engine_query_is_finished(struct storage_engine_query_handle *seqh) {

395 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");

396

397 #ifdef ENABLE_DBENGINE

@@ -346,9 +401,13 @@ static ALWAYS_INLINE int storage_engine_query_is_finished(struct storage_engine_

401 return rrddim_query_is_finished(seqh);

402 }

403

404 +// --------------------------------------------------------------------------------------------------------------------

405 +

406 void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh);

407 void rrddim_query_finalize(struct storage_engine_query_handle *seqh);

351 -static inline void storage_engine_query_finalize(struct storage_engine_query_handle *seqh) {

408 +

409 +ALWAYS_INLINE_HOT_FLATTEN

410 +static void storage_engine_query_finalize(struct storage_engine_query_handle *seqh) {

411 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");

412

413 #ifdef ENABLE_DBENGINE

@@ -359,9 +418,13 @@ static inline void storage_engine_query_finalize(struct storage_engine_query_han

418 rrddim_query_finalize(seqh);

419 }

420

421 +// --------------------------------------------------------------------------------------------------------------------

422 +

423 time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh);

424 time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *seqh);

364 -static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *seqh) {

425 +

426 +ALWAYS_INLINE_HOT_FLATTEN

427 +static time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *seqh) {

428 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");

429

430 #ifdef ENABLE_DBENGINE

src/libnetdata/aral/aral.c

+231 -143

@@ -17,7 +17,7 @@

17 // max malloc size

18 // optimal at current versions of libc is up to 256k

19 // ideal to have the same overhead as libc is 4k

20 -#define ARAL_MAX_PAGE_SIZE_MALLOC (1ULL * 1024 * 1024)

20 +#define ARAL_MAX_PAGE_SIZE_MALLOC (2ULL * 1024 * 1024) // 2MiB to use THP

22 // in malloc mode, when the page is bigger than this

23 // use anonymous private mmap pages

@@ -34,10 +34,11 @@ typedef struct aral_free {

34 } ARAL_FREE;

36 typedef struct aral_page {

37 + REFCOUNT refcount;

38 +

39 const char *filename;

40 uint8_t *data;

40 - bool marked;

42 bool started_marked;

43 bool mapped;

44 uint32_t size; // the allocation size of the page

@@ -45,17 +46,24 @@ typedef struct aral_page {

46 uint64_t elements_segmented; // fast path for acquiring new elements in this page

48 struct {

48 - uint32_t used_elements; // the number of used elements on this page

49 - uint32_t free_elements; // the number of free elements on this page

50 - uint32_t marked_elements;

51 -

49 + bool marked;

50 + struct aral_page **head_ptr;

51 struct aral_page *prev; // the prev page on the list

52 struct aral_page *next; // the next page on the list

53 } aral_lock;

55 + struct {

56 + SPINLOCK spinlock;

57 + uint32_t used_elements; // the number of used elements on this page

58 + uint32_t free_elements; // the number of free elements on this page

59 + uint32_t marked_elements;

60 + char pad[32];

61 + } page_lock;

62 +

63 struct {

64 SPINLOCK spinlock;

65 ARAL_FREE *list;

66 + char pad[40];

67 } available;

69 struct {

@@ -121,12 +129,15 @@ struct aral {

129 ARAL_PAGE *pages_marked_free; // pages with marked items and free slots

130 ARAL_PAGE *pages_marked_full; // pages with marked items completely full

131

124 - size_t user_malloc_operations;

125 - size_t user_free_operations;

132 size_t defragment_operations;

133 size_t defragment_linked_list_traversals;

134 } aral_lock;

135

136 + struct {

137 + size_t user_malloc_operations;

138 + size_t user_free_operations;

139 + } atomic;

140 +

141 struct aral_ops ops[2];

142

143 struct aral_statistics *stats;

@@ -220,6 +231,16 @@ static ALWAYS_INLINE void aral_unlock_with_trace(ARAL *ar, const char *func) {

231 #define aral_lock(ar) aral_lock_with_trace(ar, __FUNCTION__)

232 #define aral_unlock(ar) aral_unlock_with_trace(ar, __FUNCTION__)

233

234 +static ALWAYS_INLINE void aral_page_lock(ARAL *ar, ARAL_PAGE *page) {

235 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))

236 + spinlock_lock(&page->page_lock.spinlock);

237 +}

238 +

239 +static ALWAYS_INLINE void aral_page_unlock(ARAL *ar, ARAL_PAGE *page) {

240 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))

241 + spinlock_unlock(&page->page_lock.spinlock);

242 +}

243 +

244 static ALWAYS_INLINE void aral_page_available_lock(ARAL *ar, ARAL_PAGE *page) {

245 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

246 spinlock_lock(&page->available.spinlock);

@@ -230,6 +251,13 @@ static ALWAYS_INLINE void aral_page_available_unlock(ARAL *ar, ARAL_PAGE *page)

251 spinlock_unlock(&page->available.spinlock);

252 }

253

254 +static ALWAYS_INLINE bool aral_page_incoming_trylock(ARAL *ar, ARAL_PAGE *page, size_t partition) {

255 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))

256 + return spinlock_trylock(&page->incoming[partition].spinlock);

257 +

258 + return true;

259 +}

260 +

261 static ALWAYS_INLINE void aral_page_incoming_lock(ARAL *ar, ARAL_PAGE *page, size_t partition) {

262 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

263 spinlock_lock(&page->incoming[partition].spinlock);

@@ -564,7 +592,9 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_

592

593 page->size = size;

594 page->max_elements = aral_elements_in_page_size(ar, page->size);

567 - page->aral_lock.free_elements = page->max_elements;

595 + page->page_lock.free_elements = page->max_elements;

596 + spinlock_init(&page->page_lock.spinlock);

597 + page->refcount = 1;

598

599 size_t structures_size = sizeof(ARAL_PAGE) + page->max_elements * sizeof(void *);

600 size_t data_size = page->max_elements * ar->config.requested_element_size;

@@ -636,28 +666,64 @@ static void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOC

666 __atomic_sub_fetch(&ar->stats->structures.allocated_bytes, structures_size, __ATOMIC_RELAXED);

667 }

668

639 -static ALWAYS_INLINE ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

640 - size_t idx = mark_to_idx(marked);

641 - __atomic_add_fetch(&ar->ops[idx].atomic.allocators, 1, __ATOMIC_RELAXED);

669 +ALWAYS_INLINE WARNUNUSED

670 +static bool aral_page_acquire(ARAL_PAGE *page) {

671 + REFCOUNT rf = __atomic_add_fetch(&page->refcount, 1, __ATOMIC_ACQUIRE);

672 + if(rf <= 0)

673 + return false;

674 +

675 + if(rf > (REFCOUNT)page->max_elements) {

676 + __atomic_sub_fetch(&page->refcount, 1, __ATOMIC_RELAXED);

677 + return false;

678 + }

679 +

680 + return true;

681 +}

682 +

683 +ALWAYS_INLINE WARNUNUSED

684 +static ARAL_PAGE *aral_acquire_first_page(ARAL *ar, bool marked) {

685 aral_lock(ar);

686

687 ARAL_PAGE **head_ptr_free = aral_pages_head_free(ar, marked);

688 ARAL_PAGE *page = *head_ptr_free;

689

690 + if(page && !aral_page_acquire(page))

691 + page = NULL;

692 +

693 + aral_unlock(ar);

694 + return page;

695 +}

696 +

697 +ALWAYS_INLINE WARNUNUSED

698 +static bool aral_page_release(ARAL_PAGE *page) {

699 + REFCOUNT rf = __atomic_sub_fetch(&page->refcount, 1, __ATOMIC_RELEASE);

700 + if(rf == 0) {

701 + REFCOUNT expected = rf;

702 + REFCOUNT desired = REFCOUNT_DELETED;

703 + if (__atomic_compare_exchange_n(&page->refcount, &expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))

704 + return true;

705 + }

706 +

707 + return false;

708 +}

709 +

710 +static ALWAYS_INLINE ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

711 + size_t idx = mark_to_idx(marked);

712 + __atomic_add_fetch(&ar->ops[idx].atomic.allocators, 1, __ATOMIC_RELAXED);

713 +

714 #ifdef NETDATA_ARAL_INTERNAL_CHECKS

715 // bool added = false;

716 struct free_space f1, f2;

717 #endif

718

652 - while(!page || !page->aral_lock.free_elements) {

653 - internal_fatal(page && page->aral_lock.next && page->aral_lock.next->aral_lock.free_elements, "hey!");

654 -

719 + ARAL_PAGE *page;

720 + while(!(page = aral_acquire_first_page(ar, marked))) {

721 #ifdef NETDATA_ARAL_INTERNAL_CHECKS

722 f1 = check_free_space___aral_lock_needed(ar, NULL, marked);

723 #endif

724

659 - size_t page_allocation_size = 0;

725 bool can_add = false;

726 + size_t page_allocation_size = 0;

727 if(aral_adders_trylock(ar, marked)) {

728 // we can add a page - let's see it is really needed

729 size_t threads_currently_allocating = __atomic_load_n(&ar->ops[idx].atomic.allocators, __ATOMIC_RELAXED);

@@ -672,15 +738,16 @@ static ALWAYS_INLINE ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, b

738 }

739 aral_adders_unlock(ar, marked);

740 }

675 - aral_unlock(ar);

741

742 if(can_add) {

743 page = aral_create_page___no_lock_needed(ar, page_allocation_size TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);

679 - page->marked = page->started_marked = marked;

744 + page->aral_lock.marked = page->started_marked = marked;

745

746 + ARAL_PAGE **head_ptr_free = aral_pages_head_free(ar, marked);

747 aral_lock(ar);

682 -

748 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_free, page, aral_lock.prev, aral_lock.next);

749 + page->aral_lock.head_ptr = head_ptr_free;

750 + aral_unlock(ar);

751

752 //#ifdef NETDATA_ARAL_INTERNAL_CHECKS

753 // added = true;

@@ -691,24 +758,19 @@ static ALWAYS_INLINE ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, b

758 aral_adders_unlock(ar, marked);

759

760 // we have a page that is all empty

694 - // and only aral_lock() is held, so

761 // break the loop

762 break;

763 }

764 else {

765 // let the adders/deallocators do it

700 - // tinysleep();

766 sched_yield();

702 -

703 - aral_lock(ar);

704 - page = *head_ptr_free;

767 + tinysleep();

768 }

769 }

770

771 // we have a page

709 - // and aral locked

710 -

711 - internal_fatal(marked && !page->marked, "ARAL: requested a marked page, but the page found is not marked");

772 + // it is acquired

773 + // and aral is NOT locked

774

775 //#ifdef NETDATA_ARAL_INTERNAL_CHECKS

776 // if(added) {

@@ -717,42 +779,49 @@ static ALWAYS_INLINE ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, b

779 // }

780 //#endif

781

720 - internal_fatal(!page || !page->aral_lock.free_elements,

782 + internal_fatal(!page,

783 + "ARAL: '%s' failed to find a page with a free element",

784 + ar->config.name);

785 +

786 + aral_page_lock(ar, page);

787 +

788 + internal_fatal(!page->page_lock.free_elements,

789 "ARAL: '%s' selected page does not have a free slot in it",

790 ar->config.name);

791

724 - internal_fatal(page->max_elements != page->aral_lock.used_elements + page->aral_lock.free_elements,

792 + internal_fatal(page->max_elements != page->page_lock.used_elements + page->page_lock.free_elements,

793 "ARAL: '%s' page element counters do not match, "

794 "page says it can handle %zu elements, "

795 "but there are %zu used and %zu free items, "

796 "total %zu items",

797 ar->config.name,

798 (size_t)page->max_elements,

731 - (size_t)page->aral_lock.used_elements, (size_t)page->aral_lock.free_elements,

732 - (size_t)page->aral_lock.used_elements + (size_t)page->aral_lock.free_elements

733 - );

799 + (size_t)page->page_lock.used_elements, (size_t)page->page_lock.free_elements,

800 + (size_t)page->page_lock.used_elements + (size_t)page->page_lock.free_elements);

801

735 - ar->aral_lock.user_malloc_operations++;

802 + internal_fatal(page->page_lock.marked_elements > page->page_lock.used_elements,

803 + "page has more marked elements than the used ones");

804

737 - // acquire a slot for the caller

738 - page->aral_lock.used_elements++;

739 - page->aral_lock.free_elements--;

805 + page->page_lock.used_elements++;

806 + page->page_lock.free_elements--;

807

808 if(marked)

742 - page->aral_lock.marked_elements++;

743 -

744 - internal_fatal(page->aral_lock.marked_elements > page->aral_lock.used_elements,

745 - "page has more marked elements than the used ones");

809 + page->page_lock.marked_elements++;

810

747 - if(page->aral_lock.free_elements == 0) {

811 + if(unlikely(page->page_lock.used_elements == page->max_elements)) {

812 + aral_lock(ar);

813 ARAL_PAGE **head_ptr_full = aral_pages_head_full(ar, marked);

749 - internal_fatal(!is_page_in_list(*head_ptr_free, page), "Page is not in this list");

750 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*head_ptr_free, page, aral_lock.prev, aral_lock.next);

814 + internal_fatal(!is_page_in_list(*page->aral_lock.head_ptr, page), "Page is not in this list");

815 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*page->aral_lock.head_ptr, page, aral_lock.prev, aral_lock.next);

816 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_full, page, aral_lock.prev, aral_lock.next);

817 + page->aral_lock.head_ptr = head_ptr_full;

818 + aral_unlock(ar);

819 }

820

821 + aral_page_unlock(ar, page);

822 +

823 __atomic_sub_fetch(&ar->ops[idx].atomic.allocators, 1, __ATOMIC_RELAXED);

755 - aral_unlock(ar);

824 + __atomic_add_fetch(&ar->atomic.user_malloc_operations, 1, __ATOMIC_RELAXED);

825

826 return page;

827 }

@@ -774,25 +843,31 @@ static ALWAYS_INLINE void *aral_get_free_slot___no_lock_required(ARAL *ar, ARAL_

843 // Fall back to existing mechanism for reused memory

844 aral_page_available_lock(ar, page);

845

777 - if(!page->available.list) {

846 + while(!page->available.list) {

847 uint32_t bitmap = __atomic_load_n(&page->incoming_partition_bitmap, __ATOMIC_RELAXED);

779 - if(!bitmap)

848 + if (!bitmap)

849 fatal("ARAL: bitmap of incoming free elements cannot be empty at this point");

850

782 - size_t partition = __builtin_ffs((int)bitmap) - 1;

783 - // for(partition = 0; partition < ARAL_PAGE_INCOMING_PARTITIONS ; partition++) {

784 - // if (bitmap & (1U << partition))

785 - // break;

786 - // }

787 -

788 - if(partition >= ARAL_PAGE_INCOMING_PARTITIONS)

789 - fatal("ARAL: partition %zu must be smaller than %d", partition, ARAL_PAGE_INCOMING_PARTITIONS);

790 -

791 - aral_page_incoming_lock(ar, page, partition);

792 - page->available.list = page->incoming[partition].list;

793 - page->incoming[partition].list = NULL;

794 - __atomic_fetch_and(&page->incoming_partition_bitmap, ~(1U << partition), __ATOMIC_RELAXED);

795 - aral_page_incoming_unlock(ar, page, partition);

851 + while(bitmap) {

852 + size_t partition = __builtin_ffs((int)bitmap) - 1;

853 + // for(partition = 0; partition < ARAL_PAGE_INCOMING_PARTITIONS ; partition++) {

854 + // if (bitmap & (1U << partition))

855 + // break;

856 + // }

857 +

858 + if (partition >= ARAL_PAGE_INCOMING_PARTITIONS)

859 + fatal("ARAL: partition %zu must be smaller than %d", partition, ARAL_PAGE_INCOMING_PARTITIONS);

860 +

861 + if (aral_page_incoming_trylock(ar, page, partition)) {

862 + page->available.list = page->incoming[partition].list;

863 + page->incoming[partition].list = NULL;

864 + __atomic_fetch_and(&page->incoming_partition_bitmap, ~(1U << partition), __ATOMIC_RELAXED);

865 + aral_page_incoming_unlock(ar, page, partition);

866 + break;

867 + }

868 + else

869 + bitmap &= ~(1U << partition);

870 + }

871 }

872

873 ARAL_FREE *found_fr = page->available.list;

@@ -813,12 +888,20 @@ static inline void aral_add_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *pa

888 ARAL_FREE *fr = (ARAL_FREE *)ptr;

889 fr->size = ar->config.element_size;

890

816 - size_t partition = gettid_cached() % ARAL_PAGE_INCOMING_PARTITIONS;

817 - aral_page_incoming_lock(ar, page, partition);

818 - fr->next = page->incoming[partition].list;

819 - page->incoming[partition].list = fr;

820 - __atomic_fetch_or(&page->incoming_partition_bitmap, 1U << partition, __ATOMIC_RELAXED);

821 - aral_page_incoming_unlock(ar, page, partition);

891 + size_t start = gettid_cached() % ARAL_PAGE_INCOMING_PARTITIONS;

892 + while (true) {

893 + for (size_t partition = start; partition < ARAL_PAGE_INCOMING_PARTITIONS; partition++) {

894 + if (aral_page_incoming_trylock(ar, page, partition)) {

895 + fr->next = page->incoming[partition].list;

896 + page->incoming[partition].list = fr;

897 + __atomic_fetch_or(&page->incoming_partition_bitmap, 1U << partition, __ATOMIC_RELAXED);

898 + aral_page_incoming_unlock(ar, page, partition);

899 + return;

900 + }

901 + }

902 +

903 + start = 0;

904 + }

905 }

906

907 ALWAYS_INLINE void *aral_callocz_internal(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

@@ -843,30 +926,6 @@ void *aral_mallocz_internal(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEF

926 return data;

927 }

928

846 -// returns true if it moved the page to the unmarked list

847 -static ALWAYS_INLINE ARAL_PAGE **aral_remove_marked_allocation___aral_lock_needed(ARAL *ar, ARAL_PAGE **head_ptr, ARAL_PAGE *page) {

848 - internal_fatal(!page->aral_lock.marked_elements, "marked elements refcount found zero");

849 - internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");

850 -

851 - page->aral_lock.marked_elements--;

852 - if (!page->aral_lock.marked_elements && page->aral_lock.used_elements) {

853 - internal_fatal(!page->marked, "The page should be marked at this point");

854 -

855 - ARAL_PAGE **head_ptr_to = (page->aral_lock.free_elements) ? aral_pages_head_free(ar, false) : aral_pages_head_full(ar, false);

856 - internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");

857 -

858 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*head_ptr, page, aral_lock.prev, aral_lock.next);

859 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_to, page, aral_lock.prev, aral_lock.next);

860 - page->marked = false;

861 - return head_ptr_to;

862 - }

863 -

864 - internal_fatal(page->aral_lock.marked_elements > page->aral_lock.used_elements,

865 - "page has more marked elements than the used ones");

866 -

867 - return head_ptr;

868 -}

869 -

929 void aral_unmark_allocation(ARAL *ar, void *ptr) {

930 #if defined(FSANITIZE_ADDRESS)

931 return;

@@ -878,18 +937,34 @@ void aral_unmark_allocation(ARAL *ar, void *ptr) {

937 bool marked;

938 ARAL_PAGE *page = aral_get_page_pointer_after_element___do_NOT_have_aral_lock(ar, ptr, &marked);

939

881 - internal_fatal(!page->marked, "This allocation does not belong to a marked page");

940 internal_fatal(!marked, "This allocation does is not marked");

941

942 if(marked)

943 aral_set_page_pointer_after_element___do_NOT_have_aral_lock(ar, page, ptr, false);

944

887 - if(marked && page->marked) {

945 + aral_page_lock(ar, page);

946 + internal_fatal(marked && !page->page_lock.marked_elements, "Marked counter going negative.");

947 + bool unmark = marked && --page->page_lock.marked_elements == 0 && page->page_lock.used_elements;

948 +

949 + if(unmark) {

950 aral_lock(ar);

889 - ARAL_PAGE **head_ptr = page->aral_lock.free_elements ? aral_pages_head_free(ar, page->marked) : aral_pages_head_full(ar, page->marked);

890 - aral_remove_marked_allocation___aral_lock_needed(ar, head_ptr, page);

951 + internal_fatal(!is_page_in_list(*page->aral_lock.head_ptr, page), "Page is not in this list");

952 +

953 + ARAL_PAGE **head_ptr_to = (page->page_lock.free_elements) ? aral_pages_head_free(ar, false) : aral_pages_head_full(ar, false);

954 + if(page->aral_lock.head_ptr != head_ptr_to) {

955 + internal_fatal(!is_page_in_list(*page->aral_lock.head_ptr, page), "Page is not in this list");

956 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*page->aral_lock.head_ptr, page, aral_lock.prev, aral_lock.next);

957 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_to, page, aral_lock.prev, aral_lock.next);

958 + page->aral_lock.head_ptr = head_ptr_to;

959 + page->aral_lock.marked = false;

960 + }

961 +

962 + internal_fatal(page->page_lock.marked_elements > page->page_lock.used_elements,

963 + "page has more marked elements than the used ones");

964 aral_unlock(ar);

965 }

966 +

967 + aral_page_unlock(ar, page);

968 }

969

970 void aral_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

@@ -907,76 +982,91 @@ void aral_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITI

982 size_t idx = mark_to_idx(marked);

983 __atomic_add_fetch(&ar->ops[idx].atomic.deallocators, 1, __ATOMIC_RELAXED);

984

910 - aral_element_returned(ar, page);

911 -

985 // make this element available

986 aral_add_free_slot___no_lock_required(ar, page, ptr);

987

915 - aral_lock(ar);

988 + // statistic, outside the lock

989 + aral_element_returned(ar, page);

990 + __atomic_add_fetch(&ar->atomic.user_free_operations, 1, __ATOMIC_RELAXED);

991

917 - internal_fatal(!page->aral_lock.used_elements,

992 + aral_page_lock(ar, page);

993 + internal_fatal(!page->page_lock.used_elements,

994 "ARAL: '%s' pointer %p is inside a page without any active allocations.",

995 ar->config.name, ptr);

996

921 - internal_fatal(page->max_elements != page->aral_lock.used_elements + page->aral_lock.free_elements,

997 + internal_fatal(page->max_elements != page->page_lock.used_elements + page->page_lock.free_elements,

998 "ARAL: '%s' page element counters do not match, "

999 "page says it can handle %zu elements, "

1000 "but there are %zu used and %zu free items, "

1001 "total %zu items",

1002 ar->config.name,

1003 (size_t)page->max_elements,

928 - (size_t)page->aral_lock.used_elements, (size_t)page->aral_lock.free_elements,

929 - (size_t)page->aral_lock.used_elements + (size_t)page->aral_lock.free_elements

1004 + (size_t)page->page_lock.used_elements, (size_t)page->page_lock.free_elements,

1005 + (size_t)page->page_lock.used_elements + (size_t)page->page_lock.free_elements

1006 );

1007

932 - ARAL_PAGE **head_ptr = page->aral_lock.free_elements ? aral_pages_head_free(ar, page->marked) : aral_pages_head_full(ar, page->marked);

933 - internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");

934 -

935 - page->aral_lock.used_elements--;

936 - page->aral_lock.free_elements++;

1008 + page->page_lock.used_elements--;

1009 + page->page_lock.free_elements++;

1010

938 - ar->aral_lock.user_free_operations++;

1011 + internal_fatal(marked && !page->page_lock.marked_elements, "Marked counter going negative.");

1012 + bool unmark = marked && --page->page_lock.marked_elements == 0 && page->page_lock.used_elements;

1013

940 - internal_fatal(marked && !page->marked, "ARAL: found a marked element on a non-marked page");

941 -

942 - if(marked && page->marked) {

943 - head_ptr = aral_remove_marked_allocation___aral_lock_needed(ar, head_ptr, page);

944 - internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");

945 - }

1014 + internal_fatal(page->max_elements != page->page_lock.used_elements + page->page_lock.free_elements,

1015 + "ARAL: '%s' page element counters do not match, "

1016 + "page says it can handle %zu elements, "

1017 + "but there are %zu used and %zu free items, "

1018 + "total %zu items",

1019 + ar->config.name,

1020 + (size_t)page->max_elements,

1021 + (size_t)page->page_lock.used_elements, (size_t)page->page_lock.free_elements,

1022 + (size_t)page->page_lock.used_elements + (size_t)page->page_lock.free_elements);

1023

947 - internal_fatal(page->aral_lock.marked_elements > page->aral_lock.used_elements,

1024 + internal_fatal(page->page_lock.marked_elements > page->page_lock.used_elements,

1025 "page has more marked elements than the used ones");

1026

950 - // if the page is empty, release it

951 - if(unlikely(!page->aral_lock.used_elements)) {

952 - internal_fatal(page->aral_lock.marked_elements, "page has marked elements but not used ones");

953 -

954 - bool is_this_page_the_last_one = *head_ptr == page && !page->aral_lock.next;

1027 + // release it

1028 + if(unlikely(aral_page_release(page))) {

1029 + __atomic_sub_fetch(&ar->ops[idx].atomic.deallocators, 1, __ATOMIC_RELAXED);

1030

956 - if(!is_this_page_the_last_one) {

957 - internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");

958 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*head_ptr, page, aral_lock.prev, aral_lock.next);

959 - }

1031 + internal_fatal(page->page_lock.used_elements, "page has used elements but has been acquired for deletion");

1032 + internal_fatal(page->page_lock.marked_elements, "page has marked elements but not used ones");

1033

961 - __atomic_sub_fetch(&ar->ops[idx].atomic.deallocators, 1, __ATOMIC_RELAXED);

1034 + aral_lock(ar);

1035 + internal_fatal(!is_page_in_list(*page->aral_lock.head_ptr, page), "Page is not in this list");

1036 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*page->aral_lock.head_ptr, page, aral_lock.prev, aral_lock.next);

1037 aral_unlock(ar);

1038

964 - if(!is_this_page_the_last_one)

965 - aral_del_page___no_lock_needed(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);

966 -

1039 + aral_page_unlock(ar, page);

1040 + aral_del_page___no_lock_needed(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);

1041 return;

1042 }

969 - else if(page->aral_lock.free_elements) {

970 - ARAL_PAGE **head_ptr_to = aral_pages_head_free(ar, page->marked);

971 - if(head_ptr != head_ptr_to) {

972 - internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");

973 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*head_ptr, page, aral_lock.prev, aral_lock.next);

1043 + else if(unlikely(unmark)) {

1044 + aral_lock(ar);

1045 +

1046 + ARAL_PAGE **head_ptr_to = aral_pages_head_free(ar, false);

1047 + if(page->aral_lock.head_ptr != head_ptr_to) {

1048 + internal_fatal(!is_page_in_list(*page->aral_lock.head_ptr, page), "Page is not in this list");

1049 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*page->aral_lock.head_ptr, page, aral_lock.prev, aral_lock.next);

1050 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_to, page, aral_lock.prev, aral_lock.next);

1051 + page->aral_lock.head_ptr = head_ptr_to;

1052 + page->aral_lock.marked = false;

1053 + }

1054 + aral_unlock(ar);

1055 + }

1056 + else if(unlikely(page->page_lock.used_elements == page->max_elements - 1)) {

1057 + aral_lock(ar);

1058 + ARAL_PAGE **head_ptr_to = aral_pages_head_free(ar, page->aral_lock.marked);

1059 + if(page->aral_lock.head_ptr != head_ptr_to) {

1060 + internal_fatal(!is_page_in_list(*page->aral_lock.head_ptr, page), "Page is not in this list");

1061 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(*page->aral_lock.head_ptr, page, aral_lock.prev, aral_lock.next);

1062 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_to, page, aral_lock.prev, aral_lock.next);

1063 + page->aral_lock.head_ptr = head_ptr_to;

1064 }

1065 + aral_unlock(ar);

1066 }

1067

1068 + aral_page_unlock(ar, page);

1069 __atomic_sub_fetch(&ar->ops[idx].atomic.deallocators, 1, __ATOMIC_RELAXED);

979 - aral_unlock(ar);

1070 }

1071

1072 void aral_destroy_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

@@ -1321,7 +1411,7 @@ static void *aral_test_thread(void *ptr) {

1411 pointers[i] = NULL;

1412 }

1413

1324 - if (auc->single_threaded && ar->aral_lock.pages_free && ar->aral_lock.pages_free->aral_lock.used_elements) {

1414 + if (auc->single_threaded && ar->aral_lock.pages_free && ar->aral_lock.pages_free->page_lock.used_elements) {

1415 fprintf(stderr, "\n\nARAL leftovers detected (1)\n\n");

1416 __atomic_add_fetch(&auc->errors, 1, __ATOMIC_RELAXED);

1417 }

@@ -1362,7 +1452,7 @@ static void *aral_test_thread(void *ptr) {

1452 pointers[i] = NULL;

1453 }

1454

1365 - if (auc->single_threaded && ar->aral_lock.pages_free && ar->aral_lock.pages_free->aral_lock.used_elements) {

1455 + if (auc->single_threaded && ar->aral_lock.pages_free && ar->aral_lock.pages_free->page_lock.used_elements) {

1456 fprintf(stderr, "\n\nARAL leftovers detected (2)\n\n");

1457 __atomic_add_fetch(&auc->errors, 1, __ATOMIC_RELAXED);

1458 }

@@ -1410,10 +1500,8 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) {

1500 size_t countdown = seconds;

1501 while(countdown-- > 0) {

1502 sleep_usec(1 * USEC_PER_SEC);

1413 - aral_lock(auc.ar);

1414 - size_t m = auc.ar->aral_lock.user_malloc_operations;

1415 - size_t f = auc.ar->aral_lock.user_free_operations;

1416 - aral_unlock(auc.ar);

1503 + size_t m = __atomic_load_n(&auc.ar->atomic.user_malloc_operations, __ATOMIC_RELAXED);

1504 + size_t f = __atomic_load_n(&auc.ar->atomic.user_free_operations, __ATOMIC_RELAXED);

1505 fprintf(stderr, "ARAL executes %0.2f M malloc and %0.2f M free operations/s\n",

1506 (double)(m - malloc_done) / 1000000.0, (double)(f - free_done) / 1000000.0);

1507 malloc_done = m;

@@ -1434,15 +1522,15 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) {

1522

1523 usec_t ended_ut = now_monotonic_usec();

1524

1437 - if (auc.ar->aral_lock.pages_free && auc.ar->aral_lock.pages_free->aral_lock.used_elements) {

1525 + if (auc.ar->aral_lock.pages_free && auc.ar->aral_lock.pages_free->page_lock.used_elements) {

1526 fprintf(stderr, "\n\nARAL leftovers detected (3)\n\n");

1527 __atomic_add_fetch(&auc.errors, 1, __ATOMIC_RELAXED);

1528 }

1529

1530 netdata_log_info("ARAL: did %zu malloc, %zu free, "

1531 "using %zu threads, in %"PRIu64" usecs",

1444 - auc.ar->aral_lock.user_malloc_operations,

1445 - auc.ar->aral_lock.user_free_operations,

1532 + __atomic_load_n(&auc.ar->atomic.user_malloc_operations, __ATOMIC_RELAXED),

1533 + __atomic_load_n(&auc.ar->atomic.user_free_operations, __ATOMIC_RELAXED),

1534 threads,

1535 ended_ut - started_ut);

1536

@@ -1473,7 +1561,7 @@ int aral_unittest(size_t elements) {

1561

1562 aral_destroy(auc.ar);

1563

1476 - int errors = aral_stress_test(2, elements, 5);

1564 + int errors = aral_stress_test(2, elements, 10);

1565

1566 return auc.errors + errors;

1567 }

src/libnetdata/atomics/refcount.h

+20 -8

@@ -40,7 +40,8 @@ typedef int32_t REFCOUNT;

40 ((refcount) >= REFCOUNT_DELETED && (refcount) <= -REFCOUNT_MAX))

42 // returns the non-usable refcount found when it fails, the final refcount when it succeeds

43 -static ALWAYS_INLINE REFCOUNT WARNUNUSED refcount_acquire_advanced_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

43 +ALWAYS_INLINE WARNUNUSED

44 +static REFCOUNT refcount_acquire_advanced_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

45 REFCOUNT expected = refcount_references(refcount);

46 REFCOUNT desired;

@@ -60,12 +61,14 @@ static ALWAYS_INLINE REFCOUNT WARNUNUSED refcount_acquire_advanced_with_trace(RE

61 return desired;

62 }

63 -static ALWAYS_INLINE bool WARNUNUSED refcount_acquire_with_trace(REFCOUNT *refcount, const char *func) {

64 +ALWAYS_INLINE WARNUNUSED

65 +static bool refcount_acquire_with_trace(REFCOUNT *refcount, const char *func) {

66 return REFCOUNT_ACQUIRED(refcount_acquire_advanced_with_trace(refcount, func));

67 }

69 // returns the number of references remaining

68 -static ALWAYS_INLINE REFCOUNT refcount_release_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

70 +ALWAYS_INLINE

71 +static REFCOUNT refcount_release_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

72 REFCOUNT expected, desired;

74 do {

@@ -84,7 +87,8 @@ static ALWAYS_INLINE REFCOUNT refcount_release_with_trace(REFCOUNT *refcount, co

87 }

89 // returns true when the item can be deleted, false when the item is currently referenced

87 -static ALWAYS_INLINE bool WARNUNUSED refcount_acquire_for_deletion_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

90 +ALWAYS_INLINE WARNUNUSED

91 +static bool refcount_acquire_for_deletion_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

92 REFCOUNT expected = 0;

93 REFCOUNT desired = REFCOUNT_DELETED;

@@ -97,7 +101,8 @@ static ALWAYS_INLINE bool WARNUNUSED refcount_acquire_for_deletion_with_trace(RE

101 return false;

102 }

103

100 -static ALWAYS_INLINE bool WARNUNUSED refcount_release_and_acquire_for_deletion_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

104 +ALWAYS_INLINE WARNUNUSED

105 +static REFCOUNT refcount_release_and_acquire_for_deletion_advanced_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

106 REFCOUNT expected, desired;

107

108 do {

@@ -109,21 +114,27 @@ static ALWAYS_INLINE bool WARNUNUSED refcount_release_and_acquire_for_deletion_w

114 // we can get it for deletion

115 desired = REFCOUNT_DELETED;

116 if (__atomic_compare_exchange_n(refcount, &expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))

112 - return true;

117 + return desired;

118 }

119 else {

120 // we can only release it

121 desired = expected - 1;

122 if (__atomic_compare_exchange_n(refcount, &expected, desired, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED))

118 - return false;

123 + return desired;

124 }

125 } while (true);

126 }

127

128 +ALWAYS_INLINE WARNUNUSED

129 +static bool refcount_release_and_acquire_for_deletion_with_trace(REFCOUNT *refcount, const char *func __maybe_unused) {

130 + return refcount_release_and_acquire_for_deletion_advanced_with_trace(refcount, func) == REFCOUNT_DELETED;

131 +}

132 +

133 // this sleeps for 1 nanosecond (posix systems), or Sleep(0) on Windows

134 void tinysleep(void);

135

126 -static ALWAYS_INLINE bool refcount_acquire_for_deletion_and_wait_with_trace(REFCOUNT *refcount, const char *func) {

136 +ALWAYS_INLINE

137 +static bool refcount_acquire_for_deletion_and_wait_with_trace(REFCOUNT *refcount, const char *func) {

138 REFCOUNT expected = refcount_references(refcount);

139 REFCOUNT desired;

140

@@ -149,6 +160,7 @@ static ALWAYS_INLINE bool refcount_acquire_for_deletion_and_wait_with_trace(REFC

160 #define refcount_release(refcount) refcount_release_with_trace(refcount, __FUNCTION__)

161 #define refcount_acquire_for_deletion(refcount) refcount_acquire_for_deletion_with_trace(refcount, __FUNCTION__)

162 #define refcount_release_and_acquire_for_deletion(refcount) refcount_release_and_acquire_for_deletion_with_trace(refcount, __FUNCTION__)

163 +#define refcount_release_and_acquire_for_deletion_advanced(refcount) refcount_release_and_acquire_for_deletion_advanced_with_trace(refcount, __FUNCTION__)

164 #define refcount_acquire_for_deletion_and_wait(refcount) refcount_acquire_for_deletion_and_wait_with_trace(refcount, __FUNCTION__)

165

166 #endif //NETDATA_REFCOUNT_H

src/libnetdata/clocks/clocks.c

+20 -20

@@ -103,7 +103,7 @@ static __attribute__((destructor)) void clocks_fin(void) {

103 #endif

104 }

105

106 -inline time_t now_sec(clockid_t clk_id) {

106 +ALWAYS_INLINE time_t now_sec(clockid_t clk_id) {

107 struct timespec ts;

108 if(unlikely(clock_gettime(clk_id, &ts) == -1)) {

109 netdata_log_error("clock_gettime(%ld, &timespec) failed.", (long int)clk_id);

@@ -112,7 +112,7 @@ inline time_t now_sec(clockid_t clk_id) {

112 return ts.tv_sec;

113 }

114

115 -inline usec_t now_usec(clockid_t clk_id) {

115 +ALWAYS_INLINE usec_t now_usec(clockid_t clk_id) {

116 struct timespec ts;

117 if(unlikely(clock_gettime(clk_id, &ts) == -1)) {

118 netdata_log_error("clock_gettime(%ld, &timespec) failed.", (long int)clk_id);

@@ -121,7 +121,7 @@ inline usec_t now_usec(clockid_t clk_id) {

121 return (usec_t)ts.tv_sec * USEC_PER_SEC + (usec_t)(ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC;

122 }

123

124 -inline int now_timeval(clockid_t clk_id, struct timeval *tv) {

124 +ALWAYS_INLINE int now_timeval(clockid_t clk_id, struct timeval *tv) {

125 struct timespec ts;

126

127 if(unlikely(clock_gettime(clk_id, &ts) == -1)) {

@@ -136,67 +136,67 @@ inline int now_timeval(clockid_t clk_id, struct timeval *tv) {

136 return 0;

137 }

138

139 -inline time_t now_realtime_sec(void) {

139 +ALWAYS_INLINE time_t now_realtime_sec(void) {

140 return now_sec(CLOCK_REALTIME);

141 }

142

143 -inline msec_t now_realtime_msec(void) {

143 +ALWAYS_INLINE msec_t now_realtime_msec(void) {

144 return now_usec(CLOCK_REALTIME) / USEC_PER_MS;

145 }

146

147 -inline usec_t now_realtime_usec(void) {

147 +ALWAYS_INLINE usec_t now_realtime_usec(void) {

148 return now_usec(CLOCK_REALTIME);

149 }

150

151 -inline int now_realtime_timeval(struct timeval *tv) {

151 +ALWAYS_INLINE int now_realtime_timeval(struct timeval *tv) {

152 return now_timeval(CLOCK_REALTIME, tv);

153 }

154

155 -inline time_t now_monotonic_sec(void) {

155 +ALWAYS_INLINE time_t now_monotonic_sec(void) {

156 return now_sec(clock_monotonic_to_use);

157 }

158

159 -inline usec_t now_monotonic_usec(void) {

159 +ALWAYS_INLINE usec_t now_monotonic_usec(void) {

160 return now_usec(clock_monotonic_to_use);

161 }

162

163 -inline int now_monotonic_timeval(struct timeval *tv) {

163 +ALWAYS_INLINE int now_monotonic_timeval(struct timeval *tv) {

164 return now_timeval(clock_monotonic_to_use, tv);

165 }

166

167 -inline time_t now_monotonic_high_precision_sec(void) {

167 +ALWAYS_INLINE time_t now_monotonic_high_precision_sec(void) {

168 return now_sec(CLOCK_MONOTONIC);

169 }

170

171 -inline usec_t now_monotonic_high_precision_usec(void) {

171 +ALWAYS_INLINE usec_t now_monotonic_high_precision_usec(void) {

172 return now_usec(CLOCK_MONOTONIC);

173 }

174

175 -inline int now_monotonic_high_precision_timeval(struct timeval *tv) {

175 +ALWAYS_INLINE int now_monotonic_high_precision_timeval(struct timeval *tv) {

176 return now_timeval(CLOCK_MONOTONIC, tv);

177 }

178

179 -inline time_t now_boottime_sec(void) {

179 +ALWAYS_INLINE time_t now_boottime_sec(void) {

180 return now_sec(clock_boottime_to_use);

181 }

182

183 -inline usec_t now_boottime_usec(void) {

183 +ALWAYS_INLINE usec_t now_boottime_usec(void) {

184 return now_usec(clock_boottime_to_use);

185 }

186

187 -inline int now_boottime_timeval(struct timeval *tv) {

187 +ALWAYS_INLINE int now_boottime_timeval(struct timeval *tv) {

188 return now_timeval(clock_boottime_to_use, tv);

189 }

190

191 -inline usec_t timeval_usec(struct timeval *tv) {

191 +ALWAYS_INLINE usec_t timeval_usec(struct timeval *tv) {

192 return (usec_t)tv->tv_sec * USEC_PER_SEC + (tv->tv_usec % USEC_PER_SEC);

193 }

194

195 -inline msec_t timeval_msec(struct timeval *tv) {

195 +ALWAYS_INLINE msec_t timeval_msec(struct timeval *tv) {

196 return (msec_t)tv->tv_sec * MSEC_PER_SEC + ((tv->tv_usec % USEC_PER_SEC) / MSEC_PER_SEC);

197 }

198

199 -inline susec_t dt_usec_signed(struct timeval *now, struct timeval *old) {

199 +ALWAYS_INLINE susec_t dt_usec_signed(struct timeval *now, struct timeval *old) {

200 usec_t ts1 = timeval_usec(now);

201 usec_t ts2 = timeval_usec(old);

202

@@ -204,7 +204,7 @@ inline susec_t dt_usec_signed(struct timeval *now, struct timeval *old) {

204 return -((susec_t)(ts2 - ts1));

205 }

206

207 -inline usec_t dt_usec(struct timeval *now, struct timeval *old) {

207 +ALWAYS_INLINE usec_t dt_usec(struct timeval *now, struct timeval *old) {

208 usec_t ts1 = timeval_usec(now);

209 usec_t ts2 = timeval_usec(old);

210 return (ts1 > ts2) ? (ts1 - ts2) : (ts2 - ts1);

src/libnetdata/common.h

+24 -2

@@ -337,12 +337,18 @@ typedef uint32_t uid_t;

337

338 #ifdef __GNUC__

339 #define UNUSED_FUNCTION(x) __attribute__((unused)) UNUSED_##x

340 -#define ALWAYS_INLINE inline __attribute__((always_inline))

340 #define ALWAYS_INLINE_ONLY __attribute__((always_inline))

341 +#define ALWAYS_INLINE inline __attribute__((always_inline)) // Forces inlining

342 +#define ALWAYS_INLINE_HOT inline __attribute__((hot, always_inline)) // Encourages optimization and forces inlining

343 +#define ALWAYS_INLINE_HOT_FLATTEN inline __attribute__((hot, always_inline, flatten)) // Encourages optimization and forces inlining and flattening

344 +#define NOT_INLINE_HOT __attribute__((hot)) // Encourages optimization but doesn’t force inlining.

345 #else

346 #define UNUSED_FUNCTION(x) UNUSED_##x

344 -#define ALWAYS_INLINE inline

347 #define ALWAYS_INLINE_ONLY

348 +#define ALWAYS_INLINE inline

349 +#define ALWAYS_INLINE_HOT inline

350 +#define ALWAYS_INLINE_HOT_FLATTEN inline

351 +#define NOT_INLINE_HOT

352 #endif

353

354 // --------------------------------------------------------------------------------------------------------------------

@@ -364,6 +370,22 @@ typedef uint32_t uid_t;

370 a = _tmp; \

371 } while(0)

372

373 +// returns the number of times the divider fits into the total

374 +// if the divider is 0, it is treated as 1 (it returns total)

375 +#define HOWMANY(total, divider) ({ \

376 + typeof(total) _t = (total); \

377 + typeof(total) _d = (divider); \

378 + _d = _d ? _d : 1; \

379 + (_t + (_d - 1)) / _d; \

380 +})

381 +

382 +#define FIT_IN_RANGE(value, min, max) ({ \

383 + typeof(value) _v = (value); \

384 + typeof(min) _min = (min); \

385 + typeof(max) _max = (max); \

386 + (_v < _min) ? _min : ((_v > _max) ? _max : _v); \

387 +})

388 +

389 // --------------------------------------------------------------------------------------------------------------------

390 // NETDATA CLOUD

391

src/libnetdata/completion/completion.c

+9 -9

@@ -2,7 +2,7 @@

3 #include "completion.h"

5 -void completion_init(struct completion *p)

5 +ALWAYS_INLINE void completion_init(struct completion *p)

6 {

7 p->completed = 0;

8 p->completed_jobs = 0;

@@ -10,13 +10,13 @@ void completion_init(struct completion *p)

10 fatal_assert(0 == uv_mutex_init(&p->mutex));

11 }

13 -void completion_destroy(struct completion *p)

13 +ALWAYS_INLINE void completion_destroy(struct completion *p)

14 {

15 uv_cond_destroy(&p->cond);

16 uv_mutex_destroy(&p->mutex);

17 }

19 -void completion_wait_for(struct completion *p)

19 +ALWAYS_INLINE void completion_wait_for(struct completion *p)

20 {

21 uv_mutex_lock(&p->mutex);

22 while (0 == p->completed) {

@@ -26,7 +26,7 @@ void completion_wait_for(struct completion *p)

26 uv_mutex_unlock(&p->mutex);

27 }

29 -bool completion_timedwait_for(struct completion *p, uint64_t timeout_s)

29 +ALWAYS_INLINE bool completion_timedwait_for(struct completion *p, uint64_t timeout_s)

30 {

31 timeout_s *= NSEC_PER_SEC;

@@ -61,7 +61,7 @@ bool completion_timedwait_for(struct completion *p, uint64_t timeout_s)

61 return result;

62 }

64 -void completion_mark_complete(struct completion *p)

64 +ALWAYS_INLINE void completion_mark_complete(struct completion *p)

65 {

66 uv_mutex_lock(&p->mutex);

67 p->completed = 1;

@@ -69,7 +69,7 @@ void completion_mark_complete(struct completion *p)

69 uv_mutex_unlock(&p->mutex);

70 }

72 -unsigned completion_wait_for_a_job(struct completion *p, unsigned completed_jobs)

72 +ALWAYS_INLINE unsigned completion_wait_for_a_job(struct completion *p, unsigned completed_jobs)

73 {

74 uv_mutex_lock(&p->mutex);

75 while (0 == p->completed && p->completed_jobs <= completed_jobs) {

@@ -81,7 +81,7 @@ unsigned completion_wait_for_a_job(struct completion *p, unsigned completed_jobs

81 return completed_jobs;

82 }

84 -unsigned completion_wait_for_a_job_with_timeout(struct completion *p, unsigned completed_jobs, uint64_t timeout_ms)

84 +ALWAYS_INLINE unsigned completion_wait_for_a_job_with_timeout(struct completion *p, unsigned completed_jobs, uint64_t timeout_ms)

85 {

86 uint64_t timeout_ns = timeout_ms * NSEC_PER_MSEC;

87 if(!timeout_ns) timeout_ns = 1;

@@ -104,7 +104,7 @@ unsigned completion_wait_for_a_job_with_timeout(struct completion *p, unsigned c

104 return completed_jobs;

105 }

106

107 -void completion_mark_complete_a_job(struct completion *p)

107 +ALWAYS_INLINE void completion_mark_complete_a_job(struct completion *p)

108 {

109 uv_mutex_lock(&p->mutex);

110 p->completed_jobs++;

@@ -112,7 +112,7 @@ void completion_mark_complete_a_job(struct completion *p)

112 uv_mutex_unlock(&p->mutex);

113 }

114

115 -bool completion_is_done(struct completion *p)

115 +ALWAYS_INLINE bool completion_is_done(struct completion *p)

116 {

117 bool ret;

118 uv_mutex_lock(&p->mutex);

src/libnetdata/inicfg/inicfg.h

@@ -177,6 +177,8 @@ const char *inicfg_get(struct config *root, const char *section, const char *nam

177 const char *inicfg_set(struct config *root, const char *section, const char *name, const char *value);

178

179 long long inicfg_get_number(struct config *root, const char *section, const char *name, long long value);

180 +long long inicfg_get_number_range(struct config *root, const char *section, const char *name, long long value, long long min, long long max);

181 +

182 long long inicfg_set_number(struct config *root, const char *section, const char *name, long long value);

183 NETDATA_DOUBLE inicfg_get_double(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value);

184 NETDATA_DOUBLE inicfg_set_double(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value);

src/libnetdata/inicfg/inicfg_api.c

+24 -1

@@ -5,7 +5,8 @@

5 const char *inicfg_get(struct config *root, const char *section, const char *name, const char *default_value) {

6 struct config_option *opt = inicfg_get_raw_value(root, section, name, default_value, CONFIG_VALUE_TYPE_TEXT, NULL);

7 if(!opt)

8 - return default_value;

8 + // the only way for opt to be NULL, is default_value to be NULL too

9 + return NULL;

11 return string2str(opt->value);

12 }

@@ -218,6 +219,28 @@ long long inicfg_get_number(struct config *root, const char *section, const char

219 return strtoll(s, NULL, 0);

220 }

221

222 +long long inicfg_get_number_range(struct config *root, const char *section, const char *name, long long value, long long min, long long max) {

223 + char buffer[100];

224 + sprintf(buffer, "%lld", value);

225 +

226 + struct config_option *opt = inicfg_get_raw_value(root, section, name, buffer, CONFIG_VALUE_TYPE_INTEGER, NULL);

227 + if(!opt) return value;

228 +

229 + const char *s = string2str(opt->value);

230 + long long rc = strtoll(s, NULL, 0);

231 + long long rc2 = FIT_IN_RANGE(rc, min, max);

232 +

233 + if(rc != rc2) {

234 + nd_log(NDLS_DAEMON, NDLP_ERR, "CONFIG: out of range [%s].%s = %lld. Acceptable values: %lld to %lld inclusive. Setting it to %lld",

235 + section, name, rc, min, max, rc2);

236 +

237 + rc = rc2;

238 + inicfg_set_number(root, section, name, rc);

239 + }

240 +

241 + return rc;

242 +}

243 +

244 NETDATA_DOUBLE inicfg_get_double(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value) {

245 char buffer[100];

246 sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value);

src/libnetdata/inicfg/inicfg_internals.h

+7 -6

@@ -28,12 +28,13 @@ typedef enum __attribute__((packed)) {

28 } CONFIG_VALUE_TYPES;

30 typedef enum __attribute__((packed)) {

31 - CONFIG_VALUE_LOADED = (1 << 0), // has been loaded from the config

32 - CONFIG_VALUE_USED = (1 << 1), // has been accessed from the program

33 - CONFIG_VALUE_CHANGED = (1 << 2), // has been changed from the loaded value or the internal default value

34 - CONFIG_VALUE_CHECKED = (1 << 3), // has been checked if the value is different from the default

35 - CONFIG_VALUE_MIGRATED = (1 << 4), // has been migrated from an old config

36 - CONFIG_VALUE_REFORMATTED = (1 << 5), // has been reformatted with the official formatting

31 + CONFIG_VALUE_LOADED = (1 << 0), // has been loaded from the config

32 + CONFIG_VALUE_USED = (1 << 1), // has been accessed from the program

33 + CONFIG_VALUE_CHANGED = (1 << 2), // has been changed from the loaded value or the internal default value

34 + CONFIG_VALUE_CHECKED = (1 << 3), // has been checked if the value is different from the default

35 + CONFIG_VALUE_MIGRATED = (1 << 4), // has been migrated from an old config

36 + CONFIG_VALUE_REFORMATTED = (1 << 5), // has been reformatted with the official formatting

37 + CONFIG_VALUE_DEFAULT_SET = (1 << 6), // the default value has been set

38 } CONFIG_VALUE_FLAGS;

40 struct config_option {

src/libnetdata/inicfg/inicfg_options.c

+4 -2

@@ -46,6 +46,7 @@ void inicfg_option_free(struct config_option *opt) {

46 freez(opt);

47 }

49 +NEVERNULL

50 struct config_option *inicfg_option_create(struct config_section *sect, const char *name, const char *value) {

51 struct config_option *opt = callocz(1, sizeof(struct config_option));

52 opt->name = string_strdupz(name);

@@ -123,8 +124,10 @@ void inicfg_get_raw_value_of_option(struct config_option *opt, const char *defau

124 }

125 }

126

126 - if(!opt->value_default)

127 + if(!(opt->flags & CONFIG_VALUE_DEFAULT_SET)) {

128 + opt->flags |= CONFIG_VALUE_DEFAULT_SET;

129 opt->value_default = string_strdupz(default_value);

130 + }

131 }

132

133 struct config_option *inicfg_get_raw_value_of_option_in_section(struct config_section *sect, const char *option, const char *default_value, CONFIG_VALUE_TYPES type, reformat_t cb) {

@@ -134,7 +137,6 @@ struct config_option *inicfg_get_raw_value_of_option_in_section(struct config_se

137 if (!opt) {

138 if (!default_value) return NULL;

139 opt = inicfg_option_create(sect, option, default_value);

137 - if (!opt) return NULL;

140 }

141

142 inicfg_get_raw_value_of_option(opt, default_value, type, cb);

src/libnetdata/libjudy/judy-malloc.c

+2 -2

@@ -55,11 +55,11 @@ static ARAL *judy_size_aral(Word_t Words) {

56 static __thread int64_t judy_allocated = 0;

58 -void JudyAllocThreadPulseReset(void) {

58 +ALWAYS_INLINE void JudyAllocThreadPulseReset(void) {

59 judy_allocated = 0;

60 }

62 -int64_t JudyAllocThreadPulseGetAndReset(void) {

62 +ALWAYS_INLINE int64_t JudyAllocThreadPulseGetAndReset(void) {

63 int64_t rc = judy_allocated;

64 judy_allocated = 0;

65 return rc;

src/libnetdata/libnetdata.h

+2 -2

@@ -141,7 +141,7 @@ extern const char *netdata_configured_host_prefix;

141 #include "functions_evloop/functions_evloop.h"

142 #include "query_progress/progress.h"

143

144 -static inline PPvoid_t JudyLFirstThenNext(Pcvoid_t PArray, Word_t * PIndex, bool *first) {

144 +static ALWAYS_INLINE PPvoid_t JudyLFirstThenNext(Pcvoid_t PArray, Word_t * PIndex, bool *first) {

145 if(unlikely(*first)) {

146 *first = false;

147 return JudyLFirst(PArray, PIndex, PJE0);

@@ -150,7 +150,7 @@ static inline PPvoid_t JudyLFirstThenNext(Pcvoid_t PArray, Word_t * PIndex, bool

150 return JudyLNext(PArray, PIndex, PJE0);

151 }

152

153 -static inline PPvoid_t JudyLLastThenPrev(Pcvoid_t PArray, Word_t * PIndex, bool *first) {

153 +static ALWAYS_INLINE PPvoid_t JudyLLastThenPrev(Pcvoid_t PArray, Word_t * PIndex, bool *first) {

154 if(unlikely(*first)) {

155 *first = false;

156 return JudyLLast(PArray, PIndex, PJE0);

src/libnetdata/locks/locks.c

+13 -13

@@ -21,21 +21,21 @@

21 // ----------------------------------------------------------------------------

22 // mutex

24 -int __netdata_mutex_init(netdata_mutex_t *mutex) {

24 +ALWAYS_INLINE int __netdata_mutex_init(netdata_mutex_t *mutex) {

25 int ret = pthread_mutex_init(mutex, NULL);

26 if(unlikely(ret != 0))

27 netdata_log_error("MUTEX_LOCK: failed to initialize (code %d).", ret);

28 return ret;

29 }

31 -int __netdata_mutex_destroy(netdata_mutex_t *mutex) {

31 +ALWAYS_INLINE int __netdata_mutex_destroy(netdata_mutex_t *mutex) {

32 int ret = pthread_mutex_destroy(mutex);

33 if(unlikely(ret != 0))

34 netdata_log_error("MUTEX_LOCK: failed to destroy (code %d).", ret);

35 return ret;

36 }

38 -int __netdata_mutex_lock(netdata_mutex_t *mutex) {

38 +ALWAYS_INLINE int __netdata_mutex_lock(netdata_mutex_t *mutex) {

39 int ret = pthread_mutex_lock(mutex);

40 if(unlikely(ret != 0)) {

41 netdata_log_error("MUTEX_LOCK: failed to get lock (code %d)", ret);

@@ -46,7 +46,7 @@ int __netdata_mutex_lock(netdata_mutex_t *mutex) {

46 return ret;

47 }

49 -int __netdata_mutex_trylock(netdata_mutex_t *mutex) {

49 +ALWAYS_INLINE int __netdata_mutex_trylock(netdata_mutex_t *mutex) {

50 int ret = pthread_mutex_trylock(mutex);

51 if(ret != 0)

52 ;

@@ -56,7 +56,7 @@ int __netdata_mutex_trylock(netdata_mutex_t *mutex) {

56 return ret;

57 }

59 -int __netdata_mutex_unlock(netdata_mutex_t *mutex) {

59 +ALWAYS_INLINE int __netdata_mutex_unlock(netdata_mutex_t *mutex) {

60 int ret = pthread_mutex_unlock(mutex);

61 if(unlikely(ret != 0))

62 netdata_log_error("MUTEX_LOCK: failed to unlock (code %d).", ret);

@@ -146,21 +146,21 @@ int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *func

146 // ----------------------------------------------------------------------------

147 // rwlock

148

149 -int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) {

149 +ALWAYS_INLINE int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) {

150 int ret = pthread_rwlock_destroy(&rwlock->rwlock_t);

151 if(unlikely(ret != 0))

152 netdata_log_error("RW_LOCK: failed to destroy lock (code %d)", ret);

153 return ret;

154 }

155

156 -int __netdata_rwlock_init(netdata_rwlock_t *rwlock) {

156 +ALWAYS_INLINE int __netdata_rwlock_init(netdata_rwlock_t *rwlock) {

157 int ret = pthread_rwlock_init(&rwlock->rwlock_t, NULL);

158 if(unlikely(ret != 0))

159 netdata_log_error("RW_LOCK: failed to initialize lock (code %d)", ret);

160 return ret;

161 }

162

163 -int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {

163 +ALWAYS_INLINE int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {

164 int ret = pthread_rwlock_rdlock(&rwlock->rwlock_t);

165 if(unlikely(ret != 0))

166 netdata_log_error("RW_LOCK: failed to obtain read lock (code %d)", ret);

@@ -170,7 +170,7 @@ int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {

170 return ret;

171 }

172

173 -int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {

173 +ALWAYS_INLINE int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {

174 int ret = pthread_rwlock_wrlock(&rwlock->rwlock_t);

175 if(unlikely(ret != 0))

176 netdata_log_error("RW_LOCK: failed to obtain write lock (code %d)", ret);

@@ -180,7 +180,7 @@ int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {

180 return ret;

181 }

182

183 -int __netdata_rwlock_rdunlock(netdata_rwlock_t *rwlock) {

183 +ALWAYS_INLINE int __netdata_rwlock_rdunlock(netdata_rwlock_t *rwlock) {

184 int ret = pthread_rwlock_unlock(&rwlock->rwlock_t);

185 if(unlikely(ret != 0))

186 netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret);

@@ -190,7 +190,7 @@ int __netdata_rwlock_rdunlock(netdata_rwlock_t *rwlock) {

190 return ret;

191 }

192

193 -int __netdata_rwlock_wrunlock(netdata_rwlock_t *rwlock) {

193 +ALWAYS_INLINE int __netdata_rwlock_wrunlock(netdata_rwlock_t *rwlock) {

194 int ret = pthread_rwlock_unlock(&rwlock->rwlock_t);

195 if(unlikely(ret != 0))

196 netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret);

@@ -200,7 +200,7 @@ int __netdata_rwlock_wrunlock(netdata_rwlock_t *rwlock) {

200 return ret;

201 }

202

203 -int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {

203 +ALWAYS_INLINE int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {

204 int ret = pthread_rwlock_tryrdlock(&rwlock->rwlock_t);

205 if(ret != 0)

206 ;

@@ -210,7 +210,7 @@ int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {

210 return ret;

211 }

212

213 -int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {

213 +ALWAYS_INLINE int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {

214 int ret = pthread_rwlock_trywrlock(&rwlock->rwlock_t);

215 if(ret != 0)

216 ;

src/libnetdata/locks/rw-spinlock.c

+6 -6

@@ -15,7 +15,7 @@ void rw_spinlock_init_with_trace(RW_SPINLOCK *rw_spinlock, const char *func __ma

15 rw_spinlock->counter = 0;

16 }

18 -bool rw_spinlock_tryread_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

18 +ALWAYS_INLINE bool rw_spinlock_tryread_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

19 size_t spins = 0;

21 uint32_t val = __atomic_add_fetch(&rw_spinlock->counter, 1, __ATOMIC_ACQUIRE);

@@ -32,7 +32,7 @@ bool rw_spinlock_tryread_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *f

32 return true;

33 }

35 -void rw_spinlock_read_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

35 +ALWAYS_INLINE void rw_spinlock_read_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

36 size_t spins = 0;

37 usec_t usec = 1;

@@ -57,12 +57,12 @@ void rw_spinlock_read_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func

57 }

58 }

60 -void rw_spinlock_read_unlock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func __maybe_unused) {

60 +ALWAYS_INLINE void rw_spinlock_read_unlock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func __maybe_unused) {

61 __atomic_sub_fetch(&rw_spinlock->counter, 1, __ATOMIC_RELEASE);

62 nd_thread_rwspinlock_read_unlocked();

63 }

65 -bool rw_spinlock_trywrite_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

65 +ALWAYS_INLINE bool rw_spinlock_trywrite_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

66 // Optimistically set writer bit

67 uint32_t old = __atomic_fetch_or(&rw_spinlock->counter, WRITER_BIT, __ATOMIC_ACQUIRE);

@@ -85,7 +85,7 @@ bool rw_spinlock_trywrite_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *

85 return false;

86 }

88 -void rw_spinlock_write_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

88 +ALWAYS_INLINE void rw_spinlock_write_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func) {

89 size_t spins = 0;

90 usec_t usec = 1;

@@ -116,7 +116,7 @@ void rw_spinlock_write_lock_with_trace(RW_SPINLOCK *rw_spinlock, const char *fun

116 }

117 }

118

119 -void rw_spinlock_write_unlock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func __maybe_unused) {

119 +ALWAYS_INLINE void rw_spinlock_write_unlock_with_trace(RW_SPINLOCK *rw_spinlock, const char *func __maybe_unused) {

120 rw_spinlock->writer = 0;

121 __atomic_and_fetch(&rw_spinlock->counter, ~WRITER_BIT, __ATOMIC_RELEASE);

122 nd_thread_rwspinlock_write_unlocked();

src/libnetdata/locks/spinlock.c

+4 -4

@@ -10,11 +10,11 @@

11 #ifndef SPINLOCK_IMPL_WITH_MUTEX

13 -void spinlock_init_with_trace(SPINLOCK *spinlock, const char *func __maybe_unused) {

13 +ALWAYS_INLINE void spinlock_init_with_trace(SPINLOCK *spinlock, const char *func __maybe_unused) {

14 memset(spinlock, 0, sizeof(SPINLOCK));

15 }

17 -void spinlock_lock_with_trace(SPINLOCK *spinlock, const char *func) {

17 +ALWAYS_INLINE void spinlock_lock_with_trace(SPINLOCK *spinlock, const char *func) {

18 size_t spins = 0;

19 usec_t usec = 1;

@@ -40,7 +40,7 @@ void spinlock_lock_with_trace(SPINLOCK *spinlock, const char *func) {

40 worker_spinlock_contention(func, spins);

41 }

43 -void spinlock_unlock_with_trace(SPINLOCK *spinlock, const char *func __maybe_unused) {

43 +ALWAYS_INLINE void spinlock_unlock_with_trace(SPINLOCK *spinlock, const char *func __maybe_unused) {

44 #ifdef NETDATA_INTERNAL_CHECKS

45 spinlock->locker_pid = 0;

46 #endif

@@ -50,7 +50,7 @@ void spinlock_unlock_with_trace(SPINLOCK *spinlock, const char *func __maybe_unu

50 nd_thread_spinlock_unlocked();

51 }

53 -bool spinlock_trylock_with_trace(SPINLOCK *spinlock, const char *func __maybe_unused) {

53 +ALWAYS_INLINE bool spinlock_trylock_with_trace(SPINLOCK *spinlock, const char *func __maybe_unused) {

54 if (!__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) &&

55 !__atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE)) {

56 // Acquired the lock

src/libnetdata/locks/waitq.c

+9 -9

@@ -7,24 +7,24 @@

7 #define PRIORITY_SHIFT 32

8 #define NO_PRIORITY 0

10 -static inline uint64_t make_order(WAITQ_PRIORITY priority, uint64_t seqno) {

10 +static ALWAYS_INLINE uint64_t make_order(WAITQ_PRIORITY priority, uint64_t seqno) {

11 return ((uint64_t)priority << PRIORITY_SHIFT) + seqno;

12 }

14 -static inline uint64_t get_our_order(WAITQ *waitq, WAITQ_PRIORITY priority) {

14 +static ALWAYS_INLINE uint64_t get_our_order(WAITQ *waitq, WAITQ_PRIORITY priority) {

15 uint64_t seqno = __atomic_add_fetch(&waitq->last_seqno, 1, __ATOMIC_RELAXED);

16 return make_order(priority, seqno);

17 }

19 -void waitq_init(WAITQ *waitq) {

19 +ALWAYS_INLINE void waitq_init(WAITQ *waitq) {

20 spinlock_init(&waitq->spinlock);

21 waitq->current_priority = 0;

22 waitq->last_seqno = 0;

23 }

25 -void waitq_destroy(WAITQ *wq __maybe_unused) { ; }

25 +ALWAYS_INLINE void waitq_destroy(WAITQ *wq __maybe_unused) { ; }

27 -static inline bool write_our_priority(WAITQ *waitq, uint64_t our_order) {

27 +static ALWAYS_INLINE bool write_our_priority(WAITQ *waitq, uint64_t our_order) {

28 uint64_t current = __atomic_load_n(&waitq->current_priority, __ATOMIC_RELAXED);

29 if(current == our_order) return true;

@@ -44,7 +44,7 @@ static inline bool write_our_priority(WAITQ *waitq, uint64_t our_order) {

44 return true;

45 }

47 -static inline bool clear_our_priority(WAITQ *waitq, uint64_t our_order) {

47 +static ALWAYS_INLINE bool clear_our_priority(WAITQ *waitq, uint64_t our_order) {

48 uint64_t expected = our_order;

50 return

@@ -57,7 +57,7 @@ static inline bool clear_our_priority(WAITQ *waitq, uint64_t our_order) {

57 __ATOMIC_RELAXED);

58 }

60 -bool waitq_try_acquire_with_trace(WAITQ *waitq, WAITQ_PRIORITY priority, const char *func __maybe_unused) {

60 +ALWAYS_INLINE bool waitq_try_acquire_with_trace(WAITQ *waitq, WAITQ_PRIORITY priority, const char *func __maybe_unused) {

61 uint64_t our_order = get_our_order(waitq, priority);

63 bool rc = write_our_priority(waitq, our_order) && spinlock_trylock(&waitq->spinlock);

@@ -69,7 +69,7 @@ bool waitq_try_acquire_with_trace(WAITQ *waitq, WAITQ_PRIORITY priority, const c

69 return rc;

70 }

72 -void waitq_acquire_with_trace(WAITQ *waitq, WAITQ_PRIORITY priority, const char *func) {

72 +ALWAYS_INLINE void waitq_acquire_with_trace(WAITQ *waitq, WAITQ_PRIORITY priority, const char *func) {

73 uint64_t our_order = get_our_order(waitq, priority);

75 size_t spins = 0;

@@ -93,7 +93,7 @@ void waitq_acquire_with_trace(WAITQ *waitq, WAITQ_PRIORITY priority, const char

93 }

94 }

96 -void waitq_release(WAITQ *waitq) {

96 +ALWAYS_INLINE void waitq_release(WAITQ *waitq) {

97 spinlock_unlock(&waitq->spinlock);

98 }

src/libnetdata/memory/nd-mallocz.c

+7 -7

@@ -386,7 +386,7 @@ void freez_int(void *ptr, const char *file, const char *function, size_t line) {

386 }

387 #else

388

389 -char *strdupz(const char *s) {

389 +ALWAYS_INLINE char *strdupz(const char *s) {

390 char *t = strdup(s);

391 if (unlikely(!t)) {

392 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -395,7 +395,7 @@ char *strdupz(const char *s) {

395 return t;

396 }

397

398 -char *strndupz(const char *s, size_t len) {

398 +ALWAYS_INLINE char *strndupz(const char *s, size_t len) {

399 char *t = strndup(s, len);

400 if (unlikely(!t)) {

401 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -405,11 +405,11 @@ char *strndupz(const char *s, size_t len) {

405 }

406

407 // If ptr is NULL, no operation is performed.

408 -void freez(void *ptr) {

408 +ALWAYS_INLINE void freez(void *ptr) {

409 if(likely(ptr)) free(ptr);

410 }

411

412 -void *mallocz(size_t size) {

412 +ALWAYS_INLINE void *mallocz(size_t size) {

413 void *p = malloc(size);

414 if (unlikely(!p)) {

415 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -418,7 +418,7 @@ void *mallocz(size_t size) {

418 return p;

419 }

420

421 -void *callocz(size_t nmemb, size_t size) {

421 +ALWAYS_INLINE void *callocz(size_t nmemb, size_t size) {

422 void *p = calloc(nmemb, size);

423 if (unlikely(!p)) {

424 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -427,7 +427,7 @@ void *callocz(size_t nmemb, size_t size) {

427 return p;

428 }

429

430 -void *reallocz(void *ptr, size_t size) {

430 +ALWAYS_INLINE void *reallocz(void *ptr, size_t size) {

431 void *p = realloc(ptr, size);

432 if (unlikely(!p)) {

433 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -436,7 +436,7 @@ void *reallocz(void *ptr, size_t size) {

436 return p;

437 }

438

439 -void posix_memfree(void *ptr) {

439 +ALWAYS_INLINE void posix_memfree(void *ptr) {

440 free(ptr);

441 }

442 #endif

src/libnetdata/memory/nd-mmap.c

+14

@@ -112,6 +112,19 @@ inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused

112 #endif

113 }

114

115 +#define THP_SIZE (2 * 1024 * 1024) // 2 MiB THP size

116 +#define THP_MASK (THP_SIZE - 1) // Mask for alignment check

117 +

118 +inline int madvise_thp(void *mem, size_t len) {

119 +#ifdef MADV_HUGEPAGE

120 + // Check if the size is at least THP size and aligned

121 + if (len >= THP_SIZE && ((uintptr_t)mem & THP_MASK) == 0) {

122 + return madvise(mem, len, MADV_HUGEPAGE);

123 + }

124 +#endif

125 + return 0; // Do nothing if THP is not supported or size is too small

126 +}

127 +

128 int nd_munmap(void *ptr, size_t size) {

129 #ifdef NETDATA_TRACE_ALLOCATIONS

130 malloc_trace_munmap(size);

@@ -193,6 +206,7 @@ void *nd_mmap_advanced(const char *filename, size_t size, int flags, int ksm, bo

206 else netdata_log_info("Cannot seek to beginning of file '%s'.", filename);

207 }

208

209 + madvise_thp(mem, size);

210 // madvise_sequential(mem, size);

211 // madvise_dontfork(mem, size); // aral is initialized before we daemonize

212 if(dont_dump) madvise_dontdump(mem, size);

src/libnetdata/memory/nd-mmap.h

@@ -12,6 +12,7 @@ int madvise_willneed(void *mem, size_t len);

12 int madvise_dontneed(void *mem, size_t len);

13 int madvise_dontdump(void *mem, size_t len);

14 int madvise_mergeable(void *mem, size_t len);

15 +int madvise_thp(void *mem, size_t len);

17 extern size_t nd_mmap_count;

18 extern size_t nd_mmap_size;

src/libnetdata/os/sleep.c

+6 -6

@@ -3,34 +3,34 @@

3 #include "../libnetdata.h"

5 #ifdef OS_WINDOWS

6 -void tinysleep(void) {

6 +ALWAYS_INLINE void tinysleep(void) {

7 Sleep(0);

8 // SwitchToThread();

9 }

10 #else

11 -void tinysleep(void) {

11 +ALWAYS_INLINE void tinysleep(void) {

12 static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };

13 nanosleep(&ns, NULL);

14 }

15 #endif

17 #ifdef OS_WINDOWS

18 -void yield_the_processor(void) {

18 +ALWAYS_INLINE void yield_the_processor(void) {

19 Sleep(0);

20 }

21 #else

22 -void yield_the_processor(void) {

22 +ALWAYS_INLINE void yield_the_processor(void) {

23 sched_yield();

24 }

25 #endif

27 #ifdef OS_WINDOWS

28 -void microsleep(usec_t ut) {

28 +ALWAYS_INLINE void microsleep(usec_t ut) {

29 size_t ms = ut / USEC_PER_MS + ((ut == 0 || (ut % USEC_PER_MS)) ? 1 : 0);

30 Sleep(ms);

31 }

32 #else

33 -void microsleep(usec_t ut) {

33 +ALWAYS_INLINE void microsleep(usec_t ut) {

34 time_t secs = (time_t)(ut / USEC_PER_SEC);

35 nsec_t nsec = (ut % USEC_PER_SEC) * NSEC_PER_USEC + ((ut == 0) ? 1 : 0);

src/libnetdata/socket/nd-poll.c

@@ -118,6 +118,7 @@ bool nd_poll_del(nd_poll_t *ndpl, int fd) {

118 }

119

120 // Update an existing file descriptor in the event poll

121 +ALWAYS_INLINE_HOT_FLATTEN

122 bool nd_poll_upd(nd_poll_t *ndpl, int fd, nd_poll_event_t events) {

123 struct fd_info *fdi = POINTERS_GET(&ndpl->pointers, fd);

124 if(!fdi) return false;

@@ -198,6 +199,7 @@ static void sort_events(nd_poll_t *ndpl) {

199 }

200

201 // Wait for events

202 +ALWAYS_INLINE_HOT_FLATTEN

203 int nd_poll_wait(nd_poll_t *ndpl, int timeout_ms, nd_poll_result_t *result) {

204 ndpl->iteration_counter++;

205

@@ -351,6 +353,7 @@ bool nd_poll_del(nd_poll_t *ndpl, int fd) {

353 }

354

355 // Update an existing file descriptor in the event poll

356 +ALWAYS_INLINE_HOT_FLATTEN

357 bool nd_poll_upd(nd_poll_t *ndpl, int fd, nd_poll_event_t events) {

358 for (nfds_t i = 0; i < ndpl->nfds; i++) {

359 if (ndpl->fds[i].fd == fd) {

@@ -399,6 +402,7 @@ static inline void rotate_fds(nd_poll_t *ndpl) {

402 }

403

404 // Wait for events

405 +ALWAYS_INLINE_HOT_FLATTEN

406 int nd_poll_wait(nd_poll_t *ndpl, int timeout_ms, nd_poll_result_t *result) {

407 if (nd_poll_get_next_event(ndpl, result))

408 return 1; // Return immediately if there's a pending event

src/libnetdata/socket/nd-sock.c

@@ -80,6 +80,7 @@ bool nd_sock_connect_to_this(ND_SOCK *s, const char *definition, int default_por

80 return true;

81 }

83 +ALWAYS_INLINE

84 ssize_t nd_sock_send_timeout(ND_SOCK *s, void *buf, size_t len, int flags, time_t timeout) {

85 switch(wait_on_socket_or_cancel_with_timeout(&s->ssl, s->fd, (int)(timeout * 1000), POLLOUT, NULL)) {

86 case 0: // data are waiting

@@ -112,6 +113,7 @@ ssize_t nd_sock_send_timeout(ND_SOCK *s, void *buf, size_t len, int flags, time_

113 return send(s->fd, buf, len, flags);

114 }

115

116 +ALWAYS_INLINE

117 ssize_t nd_sock_recv_timeout(ND_SOCK *s, void *buf, size_t len, int flags, time_t timeout) {

118 switch(wait_on_socket_or_cancel_with_timeout(&s->ssl, s->fd, (int)(timeout * 1000), POLLIN, NULL)) {

119 case 0: // data are waiting

src/libnetdata/socket/nd-sock.h

+16 -8

@@ -50,15 +50,18 @@ static inline void nd_sock_init(ND_SOCK *s, SSL_CTX *ctx, bool verify_certificat

50 s->ctx = ctx;

51 }

53 -static inline bool nd_sock_is_ssl(ND_SOCK *s) {

53 +ALWAYS_INLINE

54 +static bool nd_sock_is_ssl(ND_SOCK *s) {

55 return SSL_connection(&s->ssl);

56 }

57 -static inline SOCKET_PEERS nd_sock_socket_peers(ND_SOCK *s) {

58 +ALWAYS_INLINE

59 +static SOCKET_PEERS nd_sock_socket_peers(ND_SOCK *s) {

60 return socket_peers(s->fd);

61 }

61 -static inline void nd_sock_close(ND_SOCK *s) {

63 +ALWAYS_INLINE

64 +static void nd_sock_close(ND_SOCK *s) {

65 netdata_ssl_close(&s->ssl);

67 if(s->fd != -1) {

@@ -69,7 +72,8 @@ static inline void nd_sock_close(ND_SOCK *s) {

72 s->error = ND_SOCK_ERR_NONE;

73 }

72 -static inline ssize_t nd_sock_read(ND_SOCK *s, void *buf, size_t num, size_t retries) {

75 +ALWAYS_INLINE

76 +static ssize_t nd_sock_read(ND_SOCK *s, void *buf, size_t num, size_t retries) {

77 ssize_t rc;

78 do {

79 if (nd_sock_is_ssl(s))

@@ -82,7 +86,8 @@ static inline ssize_t nd_sock_read(ND_SOCK *s, void *buf, size_t num, size_t ret

86 return rc;

87 }

85 -static inline ssize_t nd_sock_write(ND_SOCK *s, const void *buf, size_t num, size_t retries) {

89 +ALWAYS_INLINE

90 +static ssize_t nd_sock_write(ND_SOCK *s, const void *buf, size_t num, size_t retries) {

91 ssize_t rc;

93 do {

@@ -96,7 +101,8 @@ static inline ssize_t nd_sock_write(ND_SOCK *s, const void *buf, size_t num, siz

101 return rc;

102 }

103

99 -static inline ssize_t nd_sock_write_persist(ND_SOCK *s, const void *buf, const size_t num, size_t retries) {

104 +ALWAYS_INLINE

105 +static ssize_t nd_sock_write_persist(ND_SOCK *s, const void *buf, const size_t num, size_t retries) {

106 const uint8_t *src = (const uint8_t *)buf;

107 ssize_t bytes = 0;

108

@@ -110,14 +116,16 @@ static inline ssize_t nd_sock_write_persist(ND_SOCK *s, const void *buf, const s

116 return bytes;

117 }

118

113 -static inline ssize_t nd_sock_revc_nowait(ND_SOCK *s, void *buf, size_t num) {

119 +ALWAYS_INLINE

120 +static ssize_t nd_sock_revc_nowait(ND_SOCK *s, void *buf, size_t num) {

121 if (nd_sock_is_ssl(s))

122 return netdata_ssl_read(&s->ssl, buf, num);

123 else

124 return recv(s->fd, buf, num, MSG_DONTWAIT);

125 }

126

120 -static inline ssize_t nd_sock_send_nowait(ND_SOCK *s, void *buf, size_t num) {

127 +ALWAYS_INLINE

128 +static ssize_t nd_sock_send_nowait(ND_SOCK *s, void *buf, size_t num) {

129 if (nd_sock_is_ssl(s))

130 return netdata_ssl_write(&s->ssl, buf, num);

131 else

src/libnetdata/socket/security.c

+67 -27

@@ -24,88 +24,121 @@ static SOCKET_PEERS netdata_ssl_peers(NETDATA_SSL *ssl) {

24 static void netdata_ssl_log_error_queue(const char *call, NETDATA_SSL *ssl, unsigned long err) {

25 nd_log_limit_static_thread_var(erl, 1, 0);

27 - if(err == SSL_ERROR_NONE)

27 + if (err == SSL_ERROR_NONE)

28 err = ERR_get_error();

30 - if(err == SSL_ERROR_NONE)

30 + if (err == SSL_ERROR_NONE)

31 return;

33 - do {

34 - char *code;

33 + SOCKET_PEERS peers = netdata_ssl_peers(ssl);

34 + const char *ssl_state = ssl->conn ? SSL_state_string_long(ssl->conn) : "No SSL connection";

35 + const char *cipher = ssl->conn ? SSL_get_cipher(ssl->conn) : "Unknown";

36 + const char *alpn_proto = NULL;

37 + unsigned int alpn_len = 0;

38 +

39 +#ifdef OPENSSL_NPN_NEGOTIATED

40 + SSL_get0_alpn_selected(ssl->conn, (const unsigned char **)&alpn_proto, &alpn_len);

41 +#endif

43 + do {

44 + char *err_code;

45 switch (err) {

46 case SSL_ERROR_SSL:

38 - code = "SSL_ERROR_SSL";

47 + err_code = "SSL_ERROR_SSL";

48 ssl->state = NETDATA_SSL_STATE_FAILED;

49 break;

51 case SSL_ERROR_WANT_READ:

43 - code = "SSL_ERROR_WANT_READ";

52 + err_code = "SSL_ERROR_WANT_READ";

53 break;

55 case SSL_ERROR_WANT_WRITE:

47 - code = "SSL_ERROR_WANT_WRITE";

56 + err_code = "SSL_ERROR_WANT_WRITE";

57 break;

59 case SSL_ERROR_WANT_X509_LOOKUP:

51 - code = "SSL_ERROR_WANT_X509_LOOKUP";

60 + err_code = "SSL_ERROR_WANT_X509_LOOKUP";

61 break;

63 case SSL_ERROR_SYSCALL:

55 - code = "SSL_ERROR_SYSCALL";

64 + err_code = "SSL_ERROR_SYSCALL";

65 ssl->state = NETDATA_SSL_STATE_FAILED;

66 break;

68 case SSL_ERROR_ZERO_RETURN:

60 - code = "SSL_ERROR_ZERO_RETURN";

69 + err_code = "SSL_ERROR_ZERO_RETURN";

70 + ssl->state = NETDATA_SSL_STATE_FAILED;

71 break;

73 case SSL_ERROR_WANT_CONNECT:

64 - code = "SSL_ERROR_WANT_CONNECT";

74 + err_code = "SSL_ERROR_WANT_CONNECT";

75 break;

77 case SSL_ERROR_WANT_ACCEPT:

68 - code = "SSL_ERROR_WANT_ACCEPT";

78 + err_code = "SSL_ERROR_WANT_ACCEPT";

79 break;

81 #ifdef SSL_ERROR_WANT_ASYNC

82 case SSL_ERROR_WANT_ASYNC:

73 - code = "SSL_ERROR_WANT_ASYNC";

83 + err_code = "SSL_ERROR_WANT_ASYNC";

84 break;

85 #endif

87 #ifdef SSL_ERROR_WANT_ASYNC_JOB

88 case SSL_ERROR_WANT_ASYNC_JOB:

79 - code = "SSL_ERROR_WANT_ASYNC_JOB";

89 + err_code = "SSL_ERROR_WANT_ASYNC_JOB";

90 break;

91 #endif

93 #ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB

94 case SSL_ERROR_WANT_CLIENT_HELLO_CB:

85 - code = "SSL_ERROR_WANT_CLIENT_HELLO_CB";

95 + err_code = "SSL_ERROR_WANT_CLIENT_HELLO_CB";

96 break;

97 #endif

99 #ifdef SSL_ERROR_WANT_RETRY_VERIFY

100 case SSL_ERROR_WANT_RETRY_VERIFY:

91 - code = "SSL_ERROR_WANT_RETRY_VERIFY";

101 + err_code = "SSL_ERROR_WANT_RETRY_VERIFY";

102 break;

103 #endif

104

105 default:

96 - code = "SSL_ERROR_UNKNOWN";

106 + err_code = "SSL_ERROR_UNKNOWN";

107 break;

108 }

109

100 - char str[1024 + 1];

101 - ERR_error_string_n(err, str, 1024);

102 - str[1024] = '\0';

103 - SOCKET_PEERS peers = netdata_ssl_peers(ssl);

104 - nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR,

105 - "SSL: %s() on socket local [[%s]:%d] <-> remote [[%s]:%d], returned error %lu (%s): %s",

106 - call, peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, err, code, str);

110 + const char *reason = ERR_reason_error_string(err);

111 + int reason_code = ERR_GET_REASON(err);

112

108 - } while((err = ERR_get_error()));

113 + char err_str[1024 + 1];

114 + ERR_error_string_n(err, err_str, 1024);

115 +

116 + // Extract TLS Alert Information

117 + const char *alert_type = "None";

118 + const char *alert_desc = "None";

119 +

120 + if (ERR_GET_LIB(err) == ERR_LIB_SSL) { // Ensure it's an SSL error

121 + alert_type = SSL_alert_type_string_long(reason_code);

122 + alert_desc = SSL_alert_desc_string_long(reason_code);

123 + }

124 +

125 + nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR,

126 + "SSL ERROR: %s() on socket "

127 + "local [[%s]:%d] <-> remote [[%s]:%d], "

128 + "State [%s], Cipher: [%s], ALPN: [%.*s], "

129 + "Error [%lu, %s, %s], "

130 + "Reason [%d, %s], "

131 + "Alert [%s, %s], "

132 + "Errno [%d]",

133 + call,

134 + peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port,

135 + ssl_state, cipher, (int)alpn_len, alpn_proto ? alpn_proto : "None",

136 + err, err_code, err_str,

137 + reason_code, reason ? reason : "Unknown",

138 + alert_type, alert_desc,

139 + errno);

140 +

141 + } while ((err = ERR_get_error()));

142 }

143

144 bool netdata_ssl_open_ext(NETDATA_SSL *ssl, SSL_CTX *ctx, int fd, const unsigned char *alpn_protos, unsigned int alpn_protos_len) {

@@ -158,6 +191,7 @@ bool netdata_ssl_open(NETDATA_SSL *ssl, SSL_CTX *ctx, int fd) {

191 return netdata_ssl_open_ext(ssl, ctx, fd, NULL, 0);

192 }

193

194 +ALWAYS_INLINE

195 void netdata_ssl_close(NETDATA_SSL *ssl) {

196 errno = 0;

197 ssl->ssl_errno = 0;

@@ -177,7 +211,8 @@ void netdata_ssl_close(NETDATA_SSL *ssl) {

211 *ssl = NETDATA_SSL_UNSET_CONNECTION;

212 }

213

180 -static inline bool is_handshake_complete(NETDATA_SSL *ssl, const char *op) {

214 +ALWAYS_INLINE

215 +static bool is_handshake_complete(NETDATA_SSL *ssl, const char *op) {

216 nd_log_limit_static_thread_var(erl, 1, 0);

217

218 if(unlikely(!ssl->conn)) {

@@ -231,10 +266,12 @@ static inline bool is_handshake_complete(NETDATA_SSL *ssl, const char *op) {

266 * (These are often the same value, but can be different on some systems.)

267 */

268

269 +ALWAYS_INLINE

270 ssize_t netdata_ssl_pending(NETDATA_SSL *ssl) {

271 return SSL_pending(ssl->conn);

272 }

273

274 +ALWAYS_INLINE

275 bool netdata_ssl_has_pending(NETDATA_SSL *ssl) {

276 // this call was added on OpenSSL 1.1.0

277 // however, it is more accurate than SSL_pending()

@@ -244,6 +281,7 @@ bool netdata_ssl_has_pending(NETDATA_SSL *ssl) {

281 return SSL_pending(ssl->conn) > 0;

282 }

283

284 +ALWAYS_INLINE

285 ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num) {

286 errno = 0;

287 ssl->ssl_errno = 0;

@@ -286,6 +324,7 @@ ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num) {

324 * (These are often the same value, but can be different on some systems.)

325 */

326

327 +ALWAYS_INLINE

328 ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num) {

329 errno = 0;

330 ssl->ssl_errno = 0;

@@ -353,7 +392,8 @@ static inline bool is_handshake_initialized(NETDATA_SSL *ssl, const char *op) {

392

393 #define WANT_READ_WRITE_TIMEOUT_MS 10

394

356 -static inline bool want_read_write_should_retry(NETDATA_SSL *ssl, int err) {

395 +ALWAYS_INLINE

396 +static bool want_read_write_should_retry(NETDATA_SSL *ssl, int err) {

397 int ssl_errno = SSL_get_error(ssl->conn, err);

398 if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_WANT_WRITE) {

399 struct pollfd pfds[1] = { [0] = {

src/libnetdata/storage_number/storage_number.c

+2 -1

@@ -74,7 +74,8 @@ bool is_system_ieee754_double(void) {

74 }

75 }

77 -ALWAYS_INLINE storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {

77 +ALWAYS_INLINE_HOT_FLATTEN

78 +storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {

79 // bit 32 = sign 0:positive, 1:negative

80 // bit 31 = 0:divide, 1:multiply

81 // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)

src/libnetdata/storage_number/storage_number.h

+2 -1

@@ -130,7 +130,8 @@ static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) __attri

130 #define MAX_INCREMENTAL_PERCENT_RATE 10

131

132

133 -static ALWAYS_INLINE NETDATA_DOUBLE unpack_storage_number(storage_number value) {

133 +ALWAYS_INLINE_HOT_FLATTEN

134 +static NETDATA_DOUBLE unpack_storage_number(storage_number value) {

135 extern NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];

136

137 if(unlikely(value == SN_EMPTY_SLOT))

src/libnetdata/worker_utilization/worker_utilization.c

+25 -13

@@ -76,7 +76,7 @@ static struct workers_globals {

77 static __thread struct worker *worker = NULL; // the current thread worker

79 -static inline usec_t worker_now_monotonic_usec(void) {

79 +static ALWAYS_INLINE usec_t worker_now_monotonic_usec(void) {

80 #ifdef NETDATA_WITHOUT_WORKERS_LATENCY

81 return 0;

82 #else

@@ -200,7 +200,7 @@ void worker_unregister(void) {

200 worker = NULL;

201 }

202

203 -static inline void worker_is_idle_with_time(usec_t now) {

203 +static void worker_is_idle_with_time(usec_t now) {

204 usec_t delta = now - worker->last_action_timestamp;

205 worker->busy_time += delta;

206 worker->per_job_type[worker->job_id].worker_busy_time += delta;

@@ -213,16 +213,13 @@ static inline void worker_is_idle_with_time(usec_t now) {

213 worker->last_action_timestamp = now;

214 }

215

216 -void worker_is_idle(void) {

216 +ALWAYS_INLINE void worker_is_idle(void) {

217 if(unlikely(!worker || worker->last_action != WORKER_BUSY)) return;

218

219 worker_is_idle_with_time(worker_now_monotonic_usec());

220 }

221

222 -void worker_is_busy(size_t job_id) {

223 - if(unlikely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

224 - return;

225 -

222 +static void worker_is_busy_do(size_t job_id) {

223 usec_t now = worker_now_monotonic_usec();

224

225 if(worker->last_action == WORKER_BUSY)

@@ -238,10 +235,14 @@ void worker_is_busy(size_t job_id) {

235 worker->last_action = WORKER_BUSY;

236 }

237

241 -void worker_set_metric(size_t job_id, NETDATA_DOUBLE value) {

238 +ALWAYS_INLINE void worker_is_busy(size_t job_id) {

239 if(unlikely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

240 return;

241

242 + worker_is_busy_do(job_id);

243 +}

244 +

245 +static void worker_set_metric_do(size_t job_id, NETDATA_DOUBLE value) {

246 switch(worker->per_job_type[job_id].type) {

247 case WORKER_METRIC_INCREMENT:

248 worker->per_job_type[job_id].custom_value += value;

@@ -255,17 +256,21 @@ void worker_set_metric(size_t job_id, NETDATA_DOUBLE value) {

256 }

257 }

258

259 +ALWAYS_INLINE void worker_set_metric(size_t job_id, NETDATA_DOUBLE value) {

260 + if(unlikely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

261 + return;

262 +

263 + worker_set_metric_do(job_id, value);

264 +}

265 +

266 // --------------------------------------------------------------------------------------------------------------------

267

260 -static inline size_t pointer_hash_function(const char *func) {

268 +static ALWAYS_INLINE size_t pointer_hash_function(const char *func) {

269 uintptr_t addr = (uintptr_t)func;

270 return (size_t)(((addr >> 4) | (addr >> 16)) + func[0]) % WORKER_SPINLOCK_CONTENTION_FUNCTIONS;

271 }

272

265 -void worker_spinlock_contention(const char *func, size_t spins) {

266 - if(unlikely(!worker))

267 - return;

268 -

273 +static void worker_spinlock_contention_do(const char *func, size_t spins) {

274 size_t hash = pointer_hash_function(func);

275 for (size_t i = 0; i < WORKER_SPINLOCK_CONTENTION_FUNCTIONS; i++) {

276 size_t slot = (hash + i) % WORKER_SPINLOCK_CONTENTION_FUNCTIONS;

@@ -283,6 +288,13 @@ void worker_spinlock_contention(const char *func, size_t spins) {

288 // Array is full - do nothing

289 }

290

291 +ALWAYS_INLINE void worker_spinlock_contention(const char *func, size_t spins) {

292 + if(unlikely(!worker))

293 + return;

294 +

295 + worker_spinlock_contention_do(func, spins);

296 +}

297 +

298 // statistics interface

299

300 void workers_foreach(const char *name, void (*callback)(

src/plugins.d/pluginsd_replication.c

+1 -1

@@ -385,7 +385,7 @@ ALWAYS_INLINE PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARS

385 time_t first_entry_child = (time_t) str2ull_encoded(first_entry_child_txt);

386 time_t last_entry_child = (time_t) str2ull_encoded(last_entry_child_txt);

387

388 - bool start_streaming = (strcmp(start_streaming_txt, "true") == 0);

388 + bool start_streaming = stream_parse_enable_streaming(start_streaming_txt);

389 time_t first_entry_requested = (time_t) str2ull_encoded(first_entry_requested_txt);

390 time_t last_entry_requested = (time_t) str2ull_encoded(last_entry_requested_txt);

391

src/streaming/protocol/command-nodeid.c

+1 -4

@@ -40,10 +40,7 @@ void stream_receiver_send_node_and_claim_id_to_child(RRDHOST *host) {

40 }

42 // the sender of the child receives node id, claim id and cloud url from the receiver of the parent

43 -void stream_sender_get_node_and_claim_id_from_parent(struct sender_state *s) {

44 - char *claim_id_str = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 1);

45 - char *node_id_str = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 2);

46 - char *url = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 3);

43 +void stream_sender_get_node_and_claim_id_from_parent(struct sender_state *s, const char *claim_id_str, const char *node_id_str, const char *url) {

45 bool claimed = is_agent_claimed();

46 bool update_node_id = false;

src/streaming/protocol/commands.h

+1 -1

@@ -18,7 +18,7 @@ typedef struct rrdset_stream_buffer {

19 RRDSET_STREAM_BUFFER stream_send_metrics_init(RRDSET *st, time_t wall_clock_time);

21 -void stream_sender_get_node_and_claim_id_from_parent(struct sender_state *s);

21 +void stream_sender_get_node_and_claim_id_from_parent(struct sender_state *s, const char *claim_id_str, const char *node_id_str, const char *url);

22 void stream_receiver_send_node_and_claim_id_to_child(RRDHOST *host);

23 void stream_sender_clear_parent_claim_id(RRDHOST *host);

src/streaming/stream-compression/compression.c

+10 -10

@@ -87,25 +87,25 @@ void stream_select_receiver_compression_algorithm(struct receiver_state *rpt) {

87 }

89 bool stream_compression_initialize(struct sender_state *s) {

90 - stream_compressor_destroy(&s->compressor);

90 + stream_compressor_destroy(&s->thread.compressor);

92 // IMPORTANT

93 // KEEP THE SAME ORDER IN DECOMPRESSION

95 if(stream_has_capability(s, STREAM_CAP_ZSTD))

96 - s->compressor.algorithm = COMPRESSION_ALGORITHM_ZSTD;

96 + s->thread.compressor.algorithm = COMPRESSION_ALGORITHM_ZSTD;

97 else if(stream_has_capability(s, STREAM_CAP_LZ4))

98 - s->compressor.algorithm = COMPRESSION_ALGORITHM_LZ4;

98 + s->thread.compressor.algorithm = COMPRESSION_ALGORITHM_LZ4;

99 else if(stream_has_capability(s, STREAM_CAP_BROTLI))

100 - s->compressor.algorithm = COMPRESSION_ALGORITHM_BROTLI;

100 + s->thread.compressor.algorithm = COMPRESSION_ALGORITHM_BROTLI;

101 else if(stream_has_capability(s, STREAM_CAP_GZIP))

102 - s->compressor.algorithm = COMPRESSION_ALGORITHM_GZIP;

102 + s->thread.compressor.algorithm = COMPRESSION_ALGORITHM_GZIP;

103 else

104 - s->compressor.algorithm = COMPRESSION_ALGORITHM_NONE;

104 + s->thread.compressor.algorithm = COMPRESSION_ALGORITHM_NONE;

105

106 - if(s->compressor.algorithm != COMPRESSION_ALGORITHM_NONE) {

107 - s->compressor.level = stream_send.compression.levels[s->compressor.algorithm];

108 - stream_compressor_init(&s->compressor);

106 + if(s->thread.compressor.algorithm != COMPRESSION_ALGORITHM_NONE) {

107 + s->thread.compressor.level = stream_send.compression.levels[s->thread.compressor.algorithm];

108 + stream_compressor_init(&s->thread.compressor);

109 return true;

110 }

111

@@ -143,7 +143,7 @@ bool stream_decompression_initialize(struct receiver_state *rpt) {

143 * deactivate compression by downgrading the stream protocol.

144 */

145 void stream_compression_deactivate(struct sender_state *s) {

146 - switch(s->compressor.algorithm) {

146 + switch(s->thread.compressor.algorithm) {

147 case COMPRESSION_ALGORITHM_MAX:

148 case COMPRESSION_ALGORITHM_NONE:

149 netdata_log_error("STREAM_COMPRESSION: compression error on 'host:%s' without any compression enabled. Ignoring error.",

src/streaming/stream-conf.c

+17 -3

@@ -3,6 +3,7 @@

3 #include "database/rrd.h"

4 #include "stream-receiver-internals.h"

5 #include "stream-sender-internals.h"

6 +#include "stream-replication-sender.h"

8 static struct config stream_config = APPCONFIG_INITIALIZER;

@@ -14,6 +15,11 @@ struct _stream_send stream_send = {

16 .buffer_max_size = CBUFFER_INITIAL_MAX_SIZE,

18 + .replication = {

19 + .prefetch = 0,

20 + .threads = 0,

21 + },

22 +

23 .parents = {

24 .destination = NULL,

25 .default_port = 19999,

@@ -40,7 +46,7 @@ struct _stream_receive stream_receive = {

46 .replication = {

47 .enabled = true,

48 .period = 86400,

43 - .step = 600,

49 + .step = 3600,

50 }

51 };

@@ -140,6 +146,14 @@ void stream_conf_load() {

146 inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "replication step",

147 stream_receive.replication.step);

148

149 + stream_send.replication.threads = inicfg_get_number_range(

150 + &netdata_config, CONFIG_SECTION_DB, "replication threads",

151 + replication_threads_default(), 1, MAX_REPLICATION_THREADS);

152 +

153 + stream_send.replication.prefetch = inicfg_get_number_range(

154 + &netdata_config, CONFIG_SECTION_DB, "replication prefetch",

155 + replication_prefetch_default(), 1, MAX_REPLICATION_PREFETCH);

156 +

157 stream_send.buffer_max_size = (size_t)inicfg_get_size_bytes(

158 &stream_config, CONFIG_SECTION_STREAM, "buffer size",

159 stream_send.buffer_max_size);

@@ -296,8 +310,8 @@ void stream_conf_receiver_config(struct receiver_state *rpt, struct stream_recei

310 stream_receive.replication.period));

311

312 config->replication.step =

299 - inicfg_get_number(&stream_config, machine_guid, "replication step",

300 - inicfg_get_number(&stream_config, api_key, "replication step",

313 + inicfg_get_duration_seconds(&stream_config, machine_guid, "replication step",

314 + inicfg_get_duration_seconds(&stream_config, api_key, "replication step",

315 stream_receive.replication.step));

316

317 config->compression.enabled =

src/streaming/stream-conf.h

@@ -25,6 +25,11 @@ struct _stream_send {

26 uint32_t buffer_max_size;

28 + struct {

29 + size_t prefetch;

30 + size_t threads;

31 + } replication;

32 +

33 struct {

34 STRING *destination;

35 STRING *ssl_ca_path;

src/streaming/stream-control.c

+17 -17

@@ -14,74 +14,74 @@ static struct {

14 // --------------------------------------------------------------------------------------------------------------------

15 // backfilling

17 -static uint32_t backfill_runners(void) {

17 +ALWAYS_INLINE static uint32_t backfill_runners(void) {

18 return __atomic_load_n(&sc.backfill_runners, __ATOMIC_RELAXED);

19 }

21 -void stream_control_backfill_query_started(void) {

21 +ALWAYS_INLINE void stream_control_backfill_query_started(void) {

22 __atomic_add_fetch(&sc.backfill_runners, 1, __ATOMIC_RELAXED);

23 }

25 -void stream_control_backfill_query_finished(void) {

25 +ALWAYS_INLINE void stream_control_backfill_query_finished(void) {

26 __atomic_sub_fetch(&sc.backfill_runners, 1, __ATOMIC_RELAXED);

27 }

29 // --------------------------------------------------------------------------------------------------------------------

30 // replication

32 -static uint32_t replication_runners(void) {

32 +ALWAYS_INLINE static uint32_t replication_runners(void) {

33 return __atomic_load_n(&sc.replication_runners, __ATOMIC_RELAXED);

34 }

36 -void stream_control_replication_query_started(void) {

36 +ALWAYS_INLINE void stream_control_replication_query_started(void) {

37 __atomic_add_fetch(&sc.replication_runners, 1, __ATOMIC_RELAXED);

38 }

40 -void stream_control_replication_query_finished(void) {

40 +ALWAYS_INLINE void stream_control_replication_query_finished(void) {

41 __atomic_sub_fetch(&sc.replication_runners, 1, __ATOMIC_RELAXED);

42 }

44 // --------------------------------------------------------------------------------------------------------------------

45 // user data queries

47 -static uint32_t user_data_query_runners(void) {

47 +ALWAYS_INLINE static uint32_t user_data_query_runners(void) {

48 return __atomic_load_n(&sc.user_data_queries_runners, __ATOMIC_RELAXED);

49 }

51 -void stream_control_user_data_query_started(void) {

51 +ALWAYS_INLINE void stream_control_user_data_query_started(void) {

52 __atomic_add_fetch(&sc.user_data_queries_runners, 1, __ATOMIC_RELAXED);

53 }

55 -void stream_control_user_data_query_finished(void) {

55 +ALWAYS_INLINE void stream_control_user_data_query_finished(void) {

56 __atomic_sub_fetch(&sc.user_data_queries_runners, 1, __ATOMIC_RELAXED);

57 }

59 // --------------------------------------------------------------------------------------------------------------------

60 // user weights queries

62 -static uint32_t user_weights_query_runners(void) {

62 +ALWAYS_INLINE static uint32_t user_weights_query_runners(void) {

63 return __atomic_load_n(&sc.user_weights_queries_runners, __ATOMIC_RELAXED);

64 }

66 -void stream_control_user_weights_query_started(void) {

66 +ALWAYS_INLINE void stream_control_user_weights_query_started(void) {

67 __atomic_add_fetch(&sc.user_weights_queries_runners, 1, __ATOMIC_RELAXED);

68 }

70 -void stream_control_user_weights_query_finished(void) {

70 +ALWAYS_INLINE void stream_control_user_weights_query_finished(void) {

71 __atomic_sub_fetch(&sc.user_weights_queries_runners, 1, __ATOMIC_RELAXED);

72 }

74 // --------------------------------------------------------------------------------------------------------------------

75 // consumer API

77 -bool stream_control_ml_should_be_running(void) {

77 +ALWAYS_INLINE bool stream_control_ml_should_be_running(void) {

78 return backfill_runners() == 0 &&

79 replication_runners() == 0 &&

80 user_data_query_runners() == 0 &&

81 user_weights_query_runners() == 0;

82 }

84 -bool stream_control_children_should_be_accepted(void) {

84 +ALWAYS_INLINE bool stream_control_children_should_be_accepted(void) {

85 // we should not check for replication here.

86 // replication benefits from multiple nodes (merges the extents)

87 // and also the nodes should be close in time in the db

@@ -90,14 +90,14 @@ bool stream_control_children_should_be_accepted(void) {

90 return backfill_runners() == 0;

91 }

93 -bool stream_control_replication_should_be_running(void) {

93 +ALWAYS_INLINE bool stream_control_replication_should_be_running(void) {

94 return backfill_runners() == 0 &&

95 user_data_query_runners() == 0 &&

96 user_weights_query_runners() == 0;

97 }

99 -bool stream_control_health_should_be_running(void) {

99 +ALWAYS_INLINE bool stream_control_health_should_be_running(void) {

100 return backfill_runners() == 0 &&

101 - replication_runners() == 0 &&

101 + // replication_runners() == 0 &&

102 (user_data_query_runners() + user_weights_query_runners()) <= 1;

103 }

src/streaming/stream-receiver.c

+25 -11

@@ -110,7 +110,8 @@ static bool stream_receiver_log_transport(BUFFER *wb, void *ptr) {

110

111 // --------------------------------------------------------------------------------------------------------------------

112

113 -static inline ssize_t write_stream(struct receiver_state *r, char* buffer, size_t size) {

113 +ALWAYS_INLINE

114 +static ssize_t write_stream(struct receiver_state *r, char* buffer, size_t size) {

115 if(unlikely(!size)) {

116 internal_error(true, "%s() asked to read zero bytes", __FUNCTION__);

117 errno_clear();

@@ -132,7 +133,8 @@ static inline ssize_t write_stream(struct receiver_state *r, char* buffer, size_

133 return bytes_written;

134 }

135

135 -static inline ssize_t read_stream(struct receiver_state *r, char* buffer, size_t size) {

136 +ALWAYS_INLINE

137 +static ssize_t read_stream(struct receiver_state *r, char* buffer, size_t size) {

138 if(unlikely(!size)) {

139 internal_error(true, "%s() asked to read zero bytes", __FUNCTION__);

140 errno_clear();

@@ -156,7 +158,8 @@ static inline ssize_t read_stream(struct receiver_state *r, char* buffer, size_t

158

159 // --------------------------------------------------------------------------------------------------------------------

160

159 -static inline ssize_t receiver_read_uncompressed(struct receiver_state *r) {

161 +ALWAYS_INLINE

162 +static ssize_t receiver_read_uncompressed(struct receiver_state *r) {

163 internal_fatal(r->thread.uncompressed.read_buffer[r->thread.uncompressed.read_len] != '\0',

164 "%s: read_buffer does not start with zero #2", __FUNCTION__ );

165

@@ -192,7 +195,8 @@ static inline void receiver_move_compressed(struct receiver_state *r) {

195 }

196 }

197

195 -static inline decompressor_status_t receiver_feed_decompressor(struct receiver_state *r) {

198 +ALWAYS_INLINE_HOT_FLATTEN

199 +static decompressor_status_t receiver_feed_decompressor(struct receiver_state *r) {

200 char *buf = r->thread.compressed.buf;

201 size_t start = r->thread.compressed.start;

202 size_t signature_size = r->thread.compressed.decompressor.signature_size;

@@ -248,7 +252,8 @@ static inline decompressor_status_t receiver_feed_decompressor(struct receiver_s

252 return DECOMPRESS_OK;

253 }

254

251 -static inline decompressor_status_t receiver_get_decompressed(struct receiver_state *r) {

255 +ALWAYS_INLINE_HOT_FLATTEN

256 +static decompressor_status_t receiver_get_decompressed(struct receiver_state *r) {

257 if (unlikely(!stream_decompressed_bytes_in_buffer(&r->thread.compressed.decompressor)))

258 return DECOMPRESS_NEED_MORE_DATA;

259

@@ -272,7 +277,8 @@ static inline decompressor_status_t receiver_get_decompressed(struct receiver_st

277 return DECOMPRESS_OK;

278 }

279

275 -static inline ssize_t receiver_read_compressed(struct receiver_state *r) {

280 +ALWAYS_INLINE_HOT_FLATTEN

281 +static ssize_t receiver_read_compressed(struct receiver_state *r) {

282

283 internal_fatal(r->thread.uncompressed.read_buffer[r->thread.uncompressed.read_len] != '\0',

284 "%s: read_buffer does not start with zero #2", __FUNCTION__ );

@@ -298,7 +304,8 @@ static STREAM_HANDSHAKE receiver_set_exit_reason(struct receiver_state *rpt, STR

304 return rpt->exit.reason;

305 }

306

301 -static inline bool receiver_should_stop(struct receiver_state *rpt) {

307 +ALWAYS_INLINE

308 +static bool receiver_should_stop(struct receiver_state *rpt) {

309 if(unlikely(__atomic_load_n(&rpt->exit.shutdown, __ATOMIC_ACQUIRE))) {

310 receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP, false);

311 return true;

@@ -309,6 +316,7 @@ static inline bool receiver_should_stop(struct receiver_state *rpt) {

316

317 // --------------------------------------------------------------------------------------------------------------------

318

319 +ALWAYS_INLINE

320 void stream_receiver_handle_op(struct stream_thread *sth, struct receiver_state *rpt, struct stream_opcode *msg) {

321 ND_LOG_STACK lgs[] = {

322 ND_LOG_FIELD_STR(NDF_NIDL_NODE, rpt->host->hostname),

@@ -953,12 +961,18 @@ void stream_receiver_check_all_nodes_from_poll(struct stream_thread *sth, usec_t

961 continue;

962 }

963

956 - rpt->thread.wanted = ND_POLL_READ | (stats.bytes_outstanding ? ND_POLL_WRITE : 0);

957 - if(!nd_poll_upd(sth->run.ndpl, rpt->sock.fd, rpt->thread.wanted))

958 - nd_log(NDLS_DAEMON, NDLP_ERR,

959 - "STREAM RCV[%zu] '%s' [from %s]: failed to update nd_poll().",

964 + nd_poll_event_t wanted = ND_POLL_READ | (stats.bytes_outstanding ? ND_POLL_WRITE : 0);

965 + if(unlikely(rpt->thread.wanted != wanted)) {

966 + nd_log(NDLS_DAEMON, NDLP_DEBUG,

967 + "STREAM RCV[%zu] '%s' [from %s]: nd_poll() wanted events mismatch.",

968 sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip);

969

970 + rpt->thread.wanted = wanted;

971 + if(!nd_poll_upd(sth->run.ndpl, rpt->sock.fd, rpt->thread.wanted))

972 + nd_log(NDLS_DAEMON, NDLP_ERR,

973 + "STREAM RCV[%zu] '%s' [from %s]: failed to update nd_poll().",

974 + sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip);

975 + }

976 }

977 }

978

src/streaming/stream-replication-receiver.c

+20

@@ -259,3 +259,23 @@ bool replicate_chart_request(send_command callback, struct parser *parser, RRDHO

259 return send_replay_chart_cmd(&r, "OK", false);

260 }

261

262 +ALWAYS_INLINE bool stream_parse_enable_streaming(const char *start_streaming_txt) {

263 + bool start_streaming;

264 +

265 + if(unlikely(!start_streaming_txt || !*start_streaming_txt)) {

266 + start_streaming = false;

267 + nd_log(NDLS_DAEMON, NDLP_ERR,

268 + "REPLAY: malformed start_streaming boolean value empty");

269 + }

270 + else if(likely(strcmp(start_streaming_txt, "false") == 0))

271 + start_streaming = false;

272 + else if(likely(strcmp(start_streaming_txt, "true") == 0))

273 + start_streaming = true;

274 + else {

275 + start_streaming = false;

276 + nd_log(NDLS_DAEMON, NDLP_ERR,

277 + "REPLAY: malformed start_streaming boolean value '%s'", start_streaming_txt);

278 + }

279 +

280 + return start_streaming;

281 +}

src/streaming/stream-replication-receiver.h

@@ -20,6 +20,8 @@ bool replicate_chart_request(send_command callback, struct parser *parser,

20 time_t child_first_entry, time_t child_last_entry, time_t child_wall_clock_time,

21 time_t response_first_start_time, time_t response_last_end_time);

23 +bool stream_parse_enable_streaming(const char *start_streaming_txt);

24 +

25 #ifdef __cplusplus

26 }

27 #endif

src/streaming/stream-replication-sender.c

+84 -97

@@ -30,9 +30,6 @@

30 #define ITERATIONS_IDLE_WITHOUT_PENDING_TO_RUN_SENDER_VERIFICATION 30

31 #define SECONDS_TO_RESET_POINT_IN_TIME 10

33 -#define MAX_REPLICATION_THREADS 256

34 -#define REQUESTS_AHEAD_PER_THREAD 0 // 0 = dynamic, 1 = enable synchronous queries, > 1 static

35 -

33 static struct replication_query_statistics replication_queries = {

34 .spinlock = SPINLOCK_INITIALIZER,

35 .queries_started = 0,

@@ -105,6 +102,7 @@ struct replication_query {

102 struct replication_dimension data[];

103 };

104

105 +ALWAYS_INLINE

106 static struct replication_query *replication_query_prepare(

107 RRDSET *st,

108 time_t db_first_entry,

@@ -446,13 +444,14 @@ static bool replication_query_execute(BUFFER *wb, struct replication_query *q, s

444 q->query.before = last_end_time_in_buffer;

445 q->query.enable_streaming = false;

446

449 - internal_error(true,

450 - "STREAM SND REPLAY: current buffer size %zu is more than the "

451 - "max message size %zu for chart '%s' of host '%s'. "

452 - "Interrupting replication request (%ld to %ld, %s) at %ld to %ld, %s.",

453 - buffer_strlen(wb), max_msg_size, rrdset_id(q->st), rrdhost_hostname(q->st->rrdhost),

454 - q->request.after, q->request.before, q->request.enable_streaming?"true":"false",

455 - q->query.after, q->query.before, q->query.enable_streaming?"true":"false");

447 + internal_error(

448 + true,

449 + "STREAM SND REPLAY: current remaining sender buffer of %zu bytes cannot fit the "

450 + "message size %zu bytes for chart '%s' of host '%s'. "

451 + "Sending partial replication response %ld to %ld, %s (original: %ld to %ld, %s).",

452 + buffer_strlen(wb), max_msg_size, rrdset_id(q->st), rrdhost_hostname(q->st->rrdhost),

453 + q->query.after, q->query.before, q->query.enable_streaming?"true":"false",

454 + q->request.after, q->request.before, q->request.enable_streaming?"true":"false");

455

456 q->query.interrupted = true;

457

@@ -548,6 +547,7 @@ static bool replication_query_execute(BUFFER *wb, struct replication_query *q, s

547 return finished_with_gap;

548 }

549

550 +ALWAYS_INLINE

551 static struct replication_query *replication_response_prepare(

552 RRDSET *st,

553 bool requested_enable_streaming,

@@ -556,59 +556,44 @@ static struct replication_query *replication_response_prepare(

556 STREAM_CAPABILITIES capabilities,

557 bool synchronous

558 ) {

559 +

560 + bool query_enable_streaming = requested_enable_streaming;

561 + time_t query_after = requested_after;

562 + time_t query_before = requested_before;

563 +

564 time_t wall_clock_time = now_realtime_sec();

565

561 - if(requested_after > requested_before) {

562 - // flip them

563 - time_t t = requested_before;

564 - requested_before = requested_after;

565 - requested_after = t;

566 - }

566 + if(query_after > query_before)

567 + SWAP(query_before, query_after);

568

568 - if(requested_after > wall_clock_time) {

569 - requested_after = 0;

570 - requested_before = 0;

571 - requested_enable_streaming = true;

569 + if(!query_after || !query_before || query_after > wall_clock_time) {

570 + query_after = 0;

571 + query_before = 0;

572 + query_enable_streaming = true;

573 }

573 -

574 - if(requested_before > wall_clock_time) {

575 - requested_before = wall_clock_time;

576 - requested_enable_streaming = true;

574 + else if(query_before >= (wall_clock_time - (st->update_every * 100))) {

575 + query_before = wall_clock_time;

576 + query_enable_streaming = true;

577 }

578

579 - time_t query_after = requested_after;

580 - time_t query_before = requested_before;

581 - bool query_enable_streaming = requested_enable_streaming;

582 -

579 time_t db_first_entry = 0, db_last_entry = 0;

580 rrdset_get_retention_of_tier_for_collected_chart(

585 - st, &db_first_entry, &db_last_entry, wall_clock_time, 0);

581 + st, &db_first_entry, &db_last_entry, wall_clock_time, 0);

582

587 - if(requested_after == 0 && requested_before == 0 && requested_enable_streaming == true) {

588 - // no data requested - just enable streaming

589 - ;

590 - }

591 - else {

583 + if(query_after && query_before) {

584 if (query_after < db_first_entry)

585 query_after = db_first_entry;

586

587 if (query_before > db_last_entry)

588 query_before = db_last_entry;

589

598 - // if the parent asked us to start streaming, then fill the rest with the data that we have

599 - if (requested_enable_streaming)

600 - query_before = db_last_entry;

590 + if (query_after > query_before)

591 + SWAP(query_after, query_before);

592

602 - if (query_after > query_before) {

603 - time_t tmp = query_before;

604 - query_before = query_after;

605 - query_after = tmp;

593 + if (query_enable_streaming || query_before >= db_last_entry) {

594 + query_before = db_last_entry;

595 + query_enable_streaming = true;

596 }

607 -

608 - query_enable_streaming = (requested_enable_streaming ||

609 - query_before == db_last_entry ||

610 - !requested_after ||

611 - !requested_before) ? true : false;

597 }

598

599 return replication_query_prepare(

@@ -619,7 +604,7 @@ static struct replication_query *replication_response_prepare(

604 wall_clock_time, capabilities, synchronous);

605 }

606

622 -static void replication_response_cancel_and_finalize(struct replication_query *q) {

607 +static inline void replication_response_cancel_and_finalize(struct replication_query *q) {

608 if(!q) return;

609 replication_query_finalize(NULL, q, false);

610 }

@@ -657,7 +642,7 @@ bool replication_response_execute_finalize_and_send(struct replication_query *q,

642 if(q->query.execute)

643 finished_with_gap = replication_query_execute(wb, q, max_msg_size);

644

660 - time_t after = q->request.after;

645 + time_t after = q->query.after;

646 time_t before = q->query.before;

647 bool enable_streaming = q->query.enable_streaming;

648

@@ -804,7 +789,7 @@ static struct replication_thread {

789 size_t error_duplicate; // the number of replication requests found duplicate (same chart)

790 size_t error_flushed; // the number of replication requests deleted due to disconnections

791 size_t latest_first_time; // the 'after' timestamp of the last request we executed

807 - size_t memory; // the total memory allocated by replication

792 + int64_t memory; // the total memory allocated by replication

793 } atomic; // access should be with atomic operations

794

795 struct {

@@ -853,7 +838,7 @@ static struct replication_thread {

838 },

839 };

840

856 -size_t replication_sender_allocated_memory(void) {

841 +int64_t replication_sender_allocated_memory(void) {

842 return __atomic_load_n(&replication_globals.atomic.memory, __ATOMIC_RELAXED);

843 }

844

@@ -962,17 +947,15 @@ static void replication_sort_entry_add(struct replication_request *rq) {

947

948 Pvoid_t *inner_judy_ptr;

949

950 + JudyAllocThreadPulseReset();

951 +

952 // find the outer judy entry, using after as key

966 - size_t mem_before_outer_judyl = JudyLMemUsed(replication_globals.unsafe.queue.JudyL_array);

953 inner_judy_ptr = JudyLIns(&replication_globals.unsafe.queue.JudyL_array, (Word_t) rq->after, PJE0);

968 - size_t mem_after_outer_judyl = JudyLMemUsed(replication_globals.unsafe.queue.JudyL_array);

954 if(unlikely(!inner_judy_ptr || inner_judy_ptr == PJERR))

955 fatal("REPLICATION: corrupted outer judyL");

956

957 // add it to the inner judy, using unique_id as key

973 - size_t mem_before_inner_judyl = JudyLMemUsed(*inner_judy_ptr);

958 Pvoid_t *item = JudyLIns(inner_judy_ptr, rq->unique_id, PJE0);

975 - size_t mem_after_inner_judyl = JudyLMemUsed(*inner_judy_ptr);

959 if(unlikely(!item || item == PJERR))

960 fatal("REPLICATION: corrupted inner judyL");

961

@@ -986,7 +969,7 @@ static void replication_sort_entry_add(struct replication_request *rq) {

969

970 replication_recursive_unlock();

971

989 - __atomic_add_fetch(&replication_globals.atomic.memory, (mem_after_inner_judyl - mem_before_inner_judyl) + (mem_after_outer_judyl - mem_before_outer_judyl), __ATOMIC_RELAXED);

972 + __atomic_add_fetch(&replication_globals.atomic.memory, JudyAllocThreadPulseGetAndReset(), __ATOMIC_RELAXED);

973 }

974

975 static bool replication_sort_entry_unlink_and_free_unsafe(struct replication_sort_entry *rse, Pvoid_t **inner_judy_ppptr, bool preprocessing) {

@@ -1002,27 +985,21 @@ static bool replication_sort_entry_unlink_and_free_unsafe(struct replication_sor

985 rse->rq->indexed_in_judy = false;

986 rse->rq->not_indexed_preprocessing = preprocessing;

987

1005 - size_t memory_saved = 0;

988 + JudyAllocThreadPulseReset();

989

990 // delete it from the inner judy

1008 - size_t mem_before_inner_judyl = JudyLMemUsed(**inner_judy_ppptr);

991 JudyLDel(*inner_judy_ppptr, rse->rq->unique_id, PJE0);

1010 - size_t mem_after_inner_judyl = JudyLMemUsed(**inner_judy_ppptr);

1011 - memory_saved = mem_before_inner_judyl - mem_after_inner_judyl;

992

993 // if no items left, delete it from the outer judy

994 if(**inner_judy_ppptr == NULL) {

1015 - size_t mem_before_outer_judyl = JudyLMemUsed(replication_globals.unsafe.queue.JudyL_array);

995 JudyLDel(&replication_globals.unsafe.queue.JudyL_array, rse->rq->after, PJE0);

1017 - size_t mem_after_outer_judyl = JudyLMemUsed(replication_globals.unsafe.queue.JudyL_array);

1018 - memory_saved += mem_before_outer_judyl - mem_after_outer_judyl;

996 inner_judy_deleted = true;

997 }

998

999 // free memory

1000 replication_sort_entry_destroy(rse);

1001

1025 - __atomic_sub_fetch(&replication_globals.atomic.memory, memory_saved, __ATOMIC_RELAXED);

1002 + __atomic_add_fetch(&replication_globals.atomic.memory, JudyAllocThreadPulseGetAndReset(), __ATOMIC_RELAXED);

1003

1004 return inner_judy_deleted;

1005 }

@@ -1057,6 +1034,7 @@ static void replication_sort_entry_del(struct replication_request *rq, bool buff

1034 replication_recursive_unlock();

1035 }

1036

1037 +ALWAYS_INLINE_HOT

1038 static struct replication_request replication_request_get_first_available() {

1039 Pvoid_t *inner_judy_pptr;

1040

@@ -1210,6 +1188,7 @@ static bool sender_is_still_connected_for_this_request(struct replication_reques

1188 return rq->sender_circular_buffer_last_flush_ut == stream_circular_buffer_last_flush_ut(rq->sender->scb);

1189 }

1190

1191 +ALWAYS_INLINE_HOT

1192 static bool replication_execute_request(struct replication_request *rq, bool workers) {

1193 bool ret = false;

1194

@@ -1552,18 +1531,7 @@ static int replication_pipeline_execute_next(void) {

1531 struct replication_request *rq;

1532

1533 if(unlikely(!rtp.rqs)) {

1555 -#if REQUESTS_AHEAD_PER_THREAD == 0

1556 - rtp.max_requests_ahead = (int)netdata_conf_cpus() / 2;

1557 -

1558 - if (rtp.max_requests_ahead > libuv_worker_threads * 2)

1559 - rtp.max_requests_ahead = libuv_worker_threads * 2;

1560 -

1561 - if (rtp.max_requests_ahead < 5)

1562 - rtp.max_requests_ahead = 5;

1563 -#else

1564 - rtp.max_requests_ahead = REQUESTS_AHEAD_PER_THREAD;

1565 -#endif

1566 -

1534 + rtp.max_requests_ahead = stream_send.replication.prefetch;

1535 rtp.rqs = callocz(rtp.max_requests_ahead, sizeof(struct replication_request));

1536 __atomic_add_fetch(&replication_buffers_allocated, rtp.max_requests_ahead * sizeof(struct replication_request), __ATOMIC_RELAXED);

1537 }

@@ -1723,8 +1691,17 @@ static void replication_main_cleanup(void *pptr) {

1691 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

1692 }

1693

1694 +struct aral_statistics aral_replication_stats = { 0 };

1695 void replication_initialize(void) {

1727 - replication_globals.aral_rse = aral_by_size_acquire(sizeof(struct replication_sort_entry));

1696 + replication_globals.aral_rse = aral_create(

1697 + "replication",

1698 + sizeof(struct replication_sort_entry),

1699 + 0,

1700 + 128 * 1024, // limit it so that when replication finishes, we will not have a lot of memory lost

1701 + &aral_replication_stats,

1702 + NULL, NULL, false, false, false);

1703 +

1704 + pulse_aral_register_statistics(&aral_replication_stats, "replication");

1705 }

1706

1707 void *replication_thread_main(void *ptr) {

@@ -1732,22 +1709,7 @@ void *replication_thread_main(void *ptr) {

1709

1710 replication_initialize_workers(true);

1711

1735 - size_t threads = netdata_conf_is_parent() ? (netdata_conf_cpus() / 3) : 1;

1736 - if (threads < 1) threads = 1;

1737 - else if (threads > MAX_REPLICATION_THREADS) threads = MAX_REPLICATION_THREADS;

1738 -

1739 - threads = inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, "replication threads", threads);

1740 - if(threads < 1) {

1741 - netdata_log_error("replication threads given %zu is invalid, resetting to 1", threads);

1742 - threads = 1;

1743 - inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "replication threads", threads);

1744 - }

1745 - else if(threads > MAX_REPLICATION_THREADS) {

1746 - netdata_log_error("replication threads given %zu is invalid, resetting to %d", threads, (int)MAX_REPLICATION_THREADS);

1747 - threads = MAX_REPLICATION_THREADS;

1748 - inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "replication threads", threads);

1749 - }

1750 -

1712 + size_t threads = stream_send.replication.threads;

1713 if(--threads) {

1714 replication_globals.main_thread.threads = threads;

1715 replication_globals.main_thread.threads_ptrs = mallocz(threads * sizeof(ND_THREAD *));

@@ -1819,12 +1781,20 @@ void *replication_thread_main(void *ptr) {

1781 run_verification_countdown = ITERATIONS_IDLE_WITHOUT_PENDING_TO_RUN_SENDER_VERIFICATION;

1782 }

1783

1822 - time_t latest_first_time_t = replication_get_latest_first_time();

1823 - if(latest_first_time_t && replication_globals.unsafe.pending) {

1784 + time_t current_s = replication_get_latest_first_time();

1785 + if(current_s && replication_globals.unsafe.pending) {

1786 // completion percentage statistics

1825 - time_t now = now_realtime_sec();

1826 - time_t total = now - replication_globals.unsafe.first_time_t;

1827 - time_t done = latest_first_time_t - replication_globals.unsafe.first_time_t;

1787 + time_t now_s = now_realtime_sec();

1788 + if(current_s > now_s)

1789 + current_s = now_s;

1790 +

1791 + time_t started_s = replication_globals.unsafe.first_time_t;

1792 + if(current_s < started_s)

1793 + replication_globals.unsafe.first_time_t = started_s = current_s;

1794 +

1795 + time_t total = now_s - started_s;

1796 + time_t done = current_s - started_s;

1797 +

1798 worker_set_metric(WORKER_JOB_CUSTOM_METRIC_COMPLETION,

1799 (NETDATA_DOUBLE) done * 100.0 / (NETDATA_DOUBLE) total);

1800 }

@@ -1891,3 +1861,20 @@ void *replication_thread_main(void *ptr) {

1861

1862 return NULL;

1863 }

1864 +

1865 +int replication_threads_default(void) {

1866 + int threads = netdata_conf_is_parent() ? (int)MIN(netdata_conf_cpus(), 6) : 1;

1867 + threads = FIT_IN_RANGE(threads, 1, MAX_REPLICATION_THREADS);

1868 + return threads;

1869 +}

1870 +

1871 +int replication_prefetch_default(void) {

1872 + // Our goal is to feed the pipeline with enough requests,

1873 + // since this will allow dbengine to merge the requests that load the same extents,

1874 + // providing the best performance and minimizing disk I/O.

1875 + int target = MAX(libuv_worker_threads / 2, (int)stream_send.replication.threads * 10);

1876 +

1877 + int prefetch = (int)HOWMANY(target, stream_send.replication.threads);

1878 + prefetch = FIT_IN_RANGE(prefetch, 1, MAX_REPLICATION_PREFETCH);

1879 + return prefetch;

1880 +}

src/streaming/stream-replication-sender.h

+7 -1

@@ -10,6 +10,9 @@

10 extern "C" {

11 #endif

13 +#define MAX_REPLICATION_THREADS 256

14 +#define MAX_REPLICATION_PREFETCH 256

15 +

16 struct parser;

18 struct replication_query_statistics {

@@ -28,9 +31,12 @@ void replication_sender_delete_pending_requests(struct sender_state *sender);

31 void replication_sender_request_add(struct sender_state *sender, const char *chart_id, time_t after, time_t before, bool start_streaming);

32 void replication_sender_recalculate_buffer_used_ratio_unsafe(struct sender_state *s);

31 -size_t replication_sender_allocated_memory(void);

34 +int64_t replication_sender_allocated_memory(void);

35 size_t replication_sender_allocated_buffers(void);

37 +int replication_prefetch_default(void);

38 +int replication_threads_default(void);

39 +

40 #ifdef __cplusplus

41 }

42 #endif

src/streaming/stream-sender-api.c

+2 -2

@@ -12,7 +12,7 @@ bool stream_sender_is_connected_with_ssl(struct rrdhost *host) {

12 }

14 bool stream_sender_has_compression(struct rrdhost *host) {

15 - return host && host->sender && host->sender->compressor.initialized;

15 + return host && host->sender && host->sender->thread.compressor.initialized;

16 }

18 void stream_sender_structures_init(RRDHOST *host, bool stream, STRING *parents, STRING *api_key, STRING *send_charts_matching) {

@@ -66,7 +66,7 @@ void stream_sender_structures_free(struct rrdhost *host) {

66 stream_circular_buffer_destroy(host->sender->scb);

67 host->sender->scb = NULL;

68 waitq_destroy(&host->sender->waitq);

69 - stream_compressor_destroy(&host->sender->compressor);

69 + stream_compressor_destroy(&host->sender->thread.compressor);

71 replication_sender_cleanup(host->sender);

src/streaming/stream-sender-commit.c

+3 -3

@@ -108,7 +108,7 @@ void sender_buffer_commit(struct sender_state *s, BUFFER *wb, struct sender_buff

108 // if there are data already in the buffer, we don't need to send an opcode

109 bool enable_sending = stats->bytes_outstanding == 0;

110

111 - if (s->compressor.initialized) {

111 + if (s->thread.compressor.initialized) {

112 // compressed traffic

113 if(rrdhost_is_this_a_stream_thread(s->host))

114 worker_is_busy(WORKER_STREAM_JOB_COMPRESS);

@@ -138,14 +138,14 @@ void sender_buffer_commit(struct sender_state *s, BUFFER *wb, struct sender_buff

138 }

139

140 const char *dst;

141 - size_t dst_len = stream_compress(&s->compressor, src, size_to_compress, &dst);

141 + size_t dst_len = stream_compress(&s->thread.compressor, src, size_to_compress, &dst);

142 if (!dst_len) {

143 nd_log(NDLS_DAEMON, NDLP_ERR,

144 "STREAM SND '%s' [to %s]: COMPRESSION failed. Resetting compressor and re-trying",

145 rrdhost_hostname(s->host), s->remote_ip);

146

147 stream_compression_initialize(s);

148 - dst_len = stream_compress(&s->compressor, src, size_to_compress, &dst);

148 + dst_len = stream_compress(&s->thread.compressor, src, size_to_compress, &dst);

149 if (!dst_len)

150 goto compression_failed_with_lock;

151 }

src/streaming/stream-sender-execute.c

+73 -65

@@ -1,6 +1,7 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

3 #include "stream-thread.h"

4 +#include "stream-replication-receiver.h"

5 #include "stream-replication-sender.h"

7 struct inflight_stream_function {

@@ -93,9 +94,9 @@ struct deferred_function {

95 static void execute_deferred_function(struct sender_state *s, void *data) {

96 struct deferred_function *dfd = data;

96 - execute_commands_function(s, s->defer.end_keyword,

97 + execute_commands_function(s, s->thread.defer.end_keyword,

98 dfd->transaction, dfd->timeout_s,

98 - dfd->function, s->defer.payload,

99 + dfd->function, s->thread.defer.payload,

100 dfd->access, dfd->source);

101 }

102

@@ -103,12 +104,12 @@ static void execute_deferred_json(struct sender_state *s, void *data) {

104 const char *keyword = data;

105

106 if(strcmp(keyword, PLUGINSD_KEYWORD_JSON_CMD_STREAM_PATH) == 0)

106 - stream_path_set_from_json(s->host, buffer_tostring(s->defer.payload), true);

107 + stream_path_set_from_json(s->host, buffer_tostring(s->thread.defer.payload), true);

108 else

109 nd_log(NDLS_DAEMON, NDLP_ERR,

110 "STREAM SND '%s' [to %s]: unknown JSON keyword '%s' with payload: %s",

111 rrdhost_hostname(s->host), s->remote_ip,

111 - keyword, buffer_tostring(s->defer.payload));

112 + keyword, buffer_tostring(s->thread.defer.payload));

113 }

114

115 static void cleanup_deferred_json(struct sender_state *s __maybe_unused, void *data) {

@@ -127,15 +128,15 @@ static void cleanup_deferred_function(struct sender_state *s __maybe_unused, voi

128 }

129

130 static void cleanup_deferred_data(struct sender_state *s) {

130 - if(s->defer.cleanup)

131 - s->defer.cleanup(s, s->defer.action_data);

132 -

133 - buffer_free(s->defer.payload);

134 - s->defer.payload = NULL;

135 - s->defer.end_keyword = NULL;

136 - s->defer.action = NULL;

137 - s->defer.cleanup = NULL;

138 - s->defer.action_data = NULL;

131 + if(s->thread.defer.cleanup)

132 + s->thread.defer.cleanup(s, s->thread.defer.action_data);

133 +

134 + buffer_free(s->thread.defer.payload);

135 + s->thread.defer.payload = NULL;

136 + s->thread.defer.end_keyword = NULL;

137 + s->thread.defer.action = NULL;

138 + s->thread.defer.cleanup = NULL;

139 + s->thread.defer.action_data = NULL;

140 }

141

142 void stream_sender_execute_commands_cleanup(struct sender_state *s) {

@@ -145,7 +146,7 @@ void stream_sender_execute_commands_cleanup(struct sender_state *s) {

146 // This is just a placeholder until the gap filling state machine is inserted

147 void stream_sender_execute_commands(struct sender_state *s) {

148 ND_LOG_STACK lgs[] = {

148 - ND_LOG_FIELD_CB(NDF_REQUEST, line_splitter_reconstruct_line, &s->rbuf.line),

149 + ND_LOG_FIELD_CB(NDF_REQUEST, line_splitter_reconstruct_line, &s->thread.rbuf.line),

150 ND_LOG_FIELD_END(),

151 };

152 ND_LOG_STACK_PUSH(lgs);

@@ -155,37 +156,37 @@ void stream_sender_execute_commands(struct sender_state *s) {

156 s->log.received = buffer_create(0, NULL);

157 #endif

158

158 - char *start = s->rbuf.b, *end = &s->rbuf.b[s->rbuf.read_len], *newline;

159 + char *start = s->thread.rbuf.b, *end = &s->thread.rbuf.b[s->thread.rbuf.read_len], *newline;

160 *end = '\0';

161 for( ; start < end ; start = newline + 1) {

162 newline = strchr(start, '\n');

163

164 if(!newline) {

164 - if(s->defer.end_keyword) {

165 - buffer_strcat(s->defer.payload, start);

165 + if(s->thread.defer.end_keyword) {

166 + buffer_strcat(s->thread.defer.payload, start);

167 start = end;

168 }

169 break;

170 }

171

172 *newline = '\0';

172 - s->rbuf.line.count++;

173 + s->thread.rbuf.line.count++;

174

174 - if(s->defer.end_keyword) {

175 - if(strcmp(start, s->defer.end_keyword) == 0) {

175 + if(s->thread.defer.end_keyword) {

176 + if(strcmp(start, s->thread.defer.end_keyword) == 0) {

177 #ifdef NETDATA_LOG_STREAM_SENDER

177 - buffer_strcat(s->log.received, buffer_tostring(s->defer.payload));

178 + buffer_strcat(s->log.received, buffer_tostring(s->thread.defer.payload));

179 buffer_strcat(s->log.received, "\n");

179 - buffer_strcat(s->log.received, s->defer.end_keyword);

180 + buffer_strcat(s->log.received, s->thread.defer.end_keyword);

181 buffer_strcat(s->log.received, "\n");

182 stream_sender_log_payload(s, s->log.received, STREAM_TRAFFIC_TYPE_METADATA, true);

183 #endif

183 - s->defer.action(s, s->defer.action_data);

184 + s->thread.defer.action(s, s->thread.defer.action_data);

185 cleanup_deferred_data(s);

186 }

187 else {

187 - buffer_strcat(s->defer.payload, start);

188 - buffer_putc(s->defer.payload, '\n');

188 + buffer_strcat(s->thread.defer.payload, start);

189 + buffer_putc(s->thread.defer.payload, '\n');

190 }

191

192 continue;

@@ -197,34 +198,34 @@ void stream_sender_execute_commands(struct sender_state *s) {

198 buffer_strcat(s->log.received, "\n");

199 #endif

200

200 - s->rbuf.line.num_words = quoted_strings_splitter_whitespace(start, s->rbuf.line.words, PLUGINSD_MAX_WORDS);

201 - const char *command = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 0);

201 + s->thread.rbuf.line.num_words = quoted_strings_splitter_whitespace(start, s->thread.rbuf.line.words, PLUGINSD_MAX_WORDS);

202 + const char *command = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 0);

203

204 if(command && strcmp(command, PLUGINSD_CALL_FUNCTION) == 0) {

205 #ifdef NETDATA_LOG_STREAM_SENDER

206 stream_sender_log_payload(s, s->log.received, STREAM_TRAFFIC_TYPE_FUNCTIONS, true);

207 #endif

207 - char *transaction = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 1);

208 - char *timeout_s = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 2);

209 - char *function = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 3);

210 - char *access = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 4);

211 - char *source = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 5);

208 + char *transaction = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 1);

209 + char *timeout_s = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 2);

210 + char *function = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 3);

211 + char *access = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 4);

212 + char *source = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 5);

213

214 execute_commands_function(s, command, transaction, timeout_s, function, NULL, access, source);

215 }

216 else if(command && strcmp(command, PLUGINSD_CALL_FUNCTION_PAYLOAD_BEGIN) == 0) {

216 - char *transaction = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 1);

217 - char *timeout_s = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 2);

218 - char *function = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 3);

219 - char *access = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 4);

220 - char *source = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 5);

221 - char *content_type = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 6);

222 -

223 - s->defer.end_keyword = PLUGINSD_CALL_FUNCTION_PAYLOAD_END;

224 - s->defer.payload = buffer_create(0, NULL);

225 - s->defer.payload->content_type = content_type_string2id(content_type);

226 - s->defer.action = execute_deferred_function;

227 - s->defer.cleanup = cleanup_deferred_function;

217 + char *transaction = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 1);

218 + char *timeout_s = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 2);

219 + char *function = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 3);

220 + char *access = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 4);

221 + char *source = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 5);

222 + char *content_type = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 6);

223 +

224 + s->thread.defer.end_keyword = PLUGINSD_CALL_FUNCTION_PAYLOAD_END;

225 + s->thread.defer.payload = buffer_create(0, NULL);

226 + s->thread.defer.payload->content_type = content_type_string2id(content_type);

227 + s->thread.defer.action = execute_deferred_function;

228 + s->thread.defer.cleanup = cleanup_deferred_function;

229

230 struct deferred_function *dfd = callocz(1, sizeof(*dfd));

231 dfd->transaction = strdupz(transaction ? transaction : "");

@@ -233,7 +234,7 @@ void stream_sender_execute_commands(struct sender_state *s) {

234 dfd->access = strdupz(access ? access : "");

235 dfd->source = strdupz(source ? source : "");

236

236 - s->defer.action_data = dfd;

237 + s->thread.defer.action_data = dfd;

238 }

239 else if(command && strcmp(command, PLUGINSD_CALL_FUNCTION_CANCEL) == 0) {

240 worker_is_busy(WORKER_SENDER_JOB_EXECUTE_FUNCTION);

@@ -242,7 +243,7 @@ void stream_sender_execute_commands(struct sender_state *s) {

243 #endif

244 nd_log(NDLS_ACCESS, NDLP_DEBUG, NULL);

245

245 - char *transaction = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 1);

246 + char *transaction = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 1);

247 if(transaction && *transaction)

248 rrd_function_cancel(transaction);

249 }

@@ -253,7 +254,7 @@ void stream_sender_execute_commands(struct sender_state *s) {

254 #endif

255 nd_log(NDLS_ACCESS, NDLP_DEBUG, NULL);

256

256 - char *transaction = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 1);

257 + char *transaction = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 1);

258 if(transaction && *transaction)

259 rrd_function_progress(transaction);

260 }

@@ -268,10 +269,10 @@ void stream_sender_execute_commands(struct sender_state *s) {

269 // do not log replication commands received - way too many!

270 // nd_log(NDLS_ACCESS, NDLP_DEBUG, NULL);

271

271 - const char *chart_id = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 1);

272 - const char *start_streaming = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 2);

273 - const char *after = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 3);

274 - const char *before = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 4);

272 + const char *chart_id = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 1);

273 + const char *start_streaming = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 2);

274 + const char *after = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 3);

275 + const char *before = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 4);

276

277 if (!chart_id || !start_streaming || !after || !before) {

278 netdata_log_error("STREAM REPLAY ERROR '%s' [send to %s] %s command is incomplete"

@@ -291,7 +292,10 @@ void stream_sender_execute_commands(struct sender_state *s) {

292 #endif

293

294 replication_sender_request_add(

294 - s, chart_id, strtoll(after, NULL, 0), strtoll(before, NULL, 0), !strcmp(start_streaming, "true"));

295 + s, chart_id,

296 + strtoll(after, NULL, 0),

297 + strtoll(before, NULL, 0),

298 + stream_parse_enable_streaming(start_streaming));

299 }

300 }

301 else if(command && strcmp(command, PLUGINSD_KEYWORD_NODE_ID) == 0) {

@@ -299,33 +303,37 @@ void stream_sender_execute_commands(struct sender_state *s) {

303 #ifdef NETDATA_LOG_STREAM_SENDER

304 stream_sender_log_payload(s, s->log.received, STREAM_TRAFFIC_TYPE_METADATA, true);

305 #endif

302 - stream_sender_get_node_and_claim_id_from_parent(s);

306 + char *claim_id_str = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 1);

307 + char *node_id_str = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 2);

308 + char *url = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 3);

309 +

310 + stream_sender_get_node_and_claim_id_from_parent(s, claim_id_str, node_id_str, url);

311 }

312 else if(command && strcmp(command, PLUGINSD_KEYWORD_JSON) == 0) {

313 worker_is_busy(WORKER_SENDER_JOB_EXECUTE_META);

314

307 - char *keyword = get_word(s->rbuf.line.words, s->rbuf.line.num_words, 1);

315 + char *keyword = get_word(s->thread.rbuf.line.words, s->thread.rbuf.line.num_words, 1);

316

309 - s->defer.end_keyword = PLUGINSD_KEYWORD_JSON_END;

310 - s->defer.payload = buffer_create(0, NULL);

311 - s->defer.action = execute_deferred_json;

312 - s->defer.cleanup = cleanup_deferred_json;

313 - s->defer.action_data = strdupz(keyword);

317 + s->thread.defer.end_keyword = PLUGINSD_KEYWORD_JSON_END;

318 + s->thread.defer.payload = buffer_create(0, NULL);

319 + s->thread.defer.action = execute_deferred_json;

320 + s->thread.defer.cleanup = cleanup_deferred_json;

321 + s->thread.defer.action_data = strdupz(keyword);

322 }

323 else {

324 netdata_log_error("STREAM SND '%s' [to %s] received unknown command over connection: %s",

317 - rrdhost_hostname(s->host), s->remote_ip, s->rbuf.line.words[0]?s->rbuf.line.words[0]:"(unset)");

325 + rrdhost_hostname(s->host), s->remote_ip, s->thread.rbuf.line.words[0]?s->thread.rbuf.line.words[0]:"(unset)");

326 }

327

320 - line_splitter_reset(&s->rbuf.line);

328 + line_splitter_reset(&s->thread.rbuf.line);

329 }

330

331 if (start < end) {

324 - memmove(s->rbuf.b, start, end-start);

325 - s->rbuf.read_len = end - start;

332 + memmove(s->thread.rbuf.b, start, end-start);

333 + s->thread.rbuf.read_len = end - start;

334 }

335 else {

328 - s->rbuf.b[0] = '\0';

329 - s->rbuf.read_len = 0;

336 + s->thread.rbuf.b[0] = '\0';

337 + s->thread.rbuf.read_len = 0;

338 }

339 }

src/streaming/stream-sender-internals.h

+31 -34

@@ -32,13 +32,16 @@ typedef void (*stream_defer_cleanup_t)(struct sender_state *s, void *data);

33 struct sender_state {

34 SPINLOCK spinlock;

35 -

36 - RRDHOST *host;

35 STREAM_CAPABILITIES capabilities;

36 STREAM_CAPABILITIES disabled_capabilities;

37 int16_t hops;

40 -

38 + bool parent_using_h2o;

39 + WAITQ waitq;

40 ND_SOCK sock;

41 + RRDHOST *host;

42 +

43 + time_t last_state_since_t; // the timestamp of the last state (online/offline) change

44 + STREAM_CIRCULAR_BUFFER *scb; // sender buffer

46 struct {

47 struct stream_opcode msg; // the template for sending a message to the dispatcher - protected by sender_lock()

@@ -48,6 +51,23 @@ struct sender_state {

51 // DO NOT READ OR WRITE ANYWHERE

52 uint32_t msg_slot; // ensures a opcode queue that can never get full

54 + struct compressor_state compressor;

55 +

56 + struct {

57 + size_t size;

58 + char *b;

59 + ssize_t read_len;

60 + struct line_splitter line;

61 + } rbuf;

62 +

63 + struct {

64 + const char *end_keyword;

65 + BUFFER *payload;

66 + stream_defer_action_t action;

67 + stream_defer_cleanup_t cleanup;

68 + void *action_data;

69 + } defer;

70 +

71 nd_poll_event_t wanted;

72 usec_t last_traffic_ut;

73 struct pollfd_meta meta;

@@ -57,29 +77,6 @@ struct sender_state {

77 int8_t id; // the connector id - protected by sender_lock()

78 } connector;

60 - char remote_ip[CONNECTED_TO_SIZE + 1]; // We don't know which proxy we connect to, passed back from socket.c

61 - time_t last_state_since_t; // the timestamp of the last state (online/offline) change

62 -

63 - WAITQ waitq;

64 - STREAM_CIRCULAR_BUFFER *scb;

65 -

66 - struct {

67 - char b[PLUGINSD_LINE_MAX + 1];

68 - ssize_t read_len;

69 - struct line_splitter line;

70 - } rbuf;

71 -

72 - struct compressor_state compressor;

73 -

74 -#ifdef NETDATA_LOG_STREAM_SENDER

75 - struct {

76 - SPINLOCK spinlock;

77 - struct timespec first_call;

78 - BUFFER *received;

79 - FILE *fp;

80 - } log;

81 -#endif

82 -

80 struct {

81 bool shutdown; // when set, the sender should stop sending this host

82 STREAM_HANDSHAKE reason; // the reason we decided to stop this sender

@@ -99,18 +96,18 @@ struct sender_state {

96 size_t charts_replicating; // the number of unique charts having pending replication requests (on every request one is added and is removed when we finish it - it does not track completion of the replication for this chart)

97 bool reached_max; // true when the sender buffer should not get more replication responses

98 } atomic;

102 -

99 } replication;

100

101 +#ifdef NETDATA_LOG_STREAM_SENDER

102 struct {

106 - const char *end_keyword;

107 - BUFFER *payload;

108 - stream_defer_action_t action;

109 - stream_defer_cleanup_t cleanup;

110 - void *action_data;

111 - } defer;

103 + SPINLOCK spinlock;

104 + struct timespec first_call;

105 + BUFFER *received;

106 + FILE *fp;

107 + } log;

108 +#endif

109

113 - bool parent_using_h2o;

110 + char remote_ip[CONNECTED_TO_SIZE + 1]; // We don't know which proxy we connect to, passed back from socket.c

111 };

112

113 #define stream_sender_lock(sender) spinlock_lock(&(sender)->spinlock)

src/streaming/stream-sender.c

+25 -7

@@ -130,7 +130,12 @@ void stream_sender_on_connect(struct sender_state *s) {

130 stream_sender_on_connect_and_disconnect(s);

131

132 s->thread.last_traffic_ut = now_monotonic_usec();

133 - s->rbuf.read_len = 0;

133 +

134 + freez(s->thread.rbuf.b);

135 + s->thread.rbuf.size = PLUGINSD_LINE_MAX + 1;

136 + s->thread.rbuf.b = mallocz(s->thread.rbuf.size);

137 + s->thread.rbuf.b[0] = '\0';

138 + s->thread.rbuf.read_len = 0;

139 }

140

141 static void stream_sender_on_ready_to_dispatch(struct sender_state *s) {

@@ -159,6 +164,11 @@ void stream_sender_on_disconnect(struct sender_state *s) {

164 // update the child (the receiver side) for this parent

165 stream_path_parent_disconnected(s->host);

166 stream_receiver_send_node_and_claim_id_to_child(s->host);

167 +

168 + freez(s->thread.rbuf.b);

169 + s->thread.rbuf.size = 0;

170 + s->thread.rbuf.b = NULL;

171 + s->thread.rbuf.read_len = 0;

172 }

173

174 // --------------------------------------------------------------------------------------------------------------------

@@ -204,6 +214,7 @@ static bool stream_sender_log_dst_port(BUFFER *wb, void *ptr) {

214 // --------------------------------------------------------------------------------------------------------------------

215 // opcodes

216

217 +ALWAYS_INLINE

218 void stream_sender_handle_op(struct stream_thread *sth, struct sender_state *s, struct stream_opcode *msg) {

219 ND_LOG_STACK lgs[] = {

220 ND_LOG_FIELD_STR(NDF_NIDL_NODE, s->host->hostname),

@@ -508,11 +519,18 @@ void stream_sender_check_all_nodes_from_poll(struct stream_thread *sth, usec_t n

519 bytes_compressed += stats.bytes_added;

520 bytes_uncompressed += stats.bytes_uncompressed;

521

511 - s->thread.wanted = ND_POLL_READ | (stats.bytes_outstanding ? ND_POLL_WRITE : 0);

512 - if(!nd_poll_upd(sth->run.ndpl, s->sock.fd, s->thread.wanted))

513 - nd_log(NDLS_DAEMON, NDLP_ERR,

514 - "STREAM SND[%zu] '%s' [to %s]: failed to update nd_poll().",

522 + nd_poll_event_t wanted = ND_POLL_READ | (stats.bytes_outstanding ? ND_POLL_WRITE : 0);

523 + if(unlikely(s->thread.wanted != wanted)) {

524 + nd_log(NDLS_DAEMON, NDLP_DEBUG,

525 + "STREAM SND[%zu] '%s' [to %s]: nd_poll() wanted events mismatch.",

526 sth->id, rrdhost_hostname(s->host), s->remote_ip);

527 +

528 + s->thread.wanted = wanted;

529 + if(!nd_poll_upd(sth->run.ndpl, s->sock.fd, s->thread.wanted))

530 + nd_log(NDLS_DAEMON, NDLP_ERR,

531 + "STREAM SND[%zu] '%s' [to %s]: failed to update nd_poll().",

532 + sth->id, rrdhost_hostname(s->host), s->remote_ip);

533 + }

534 }

535

536 if (bytes_compressed && bytes_uncompressed) {

@@ -724,9 +742,9 @@ bool stream_sender_send_data(struct stream_thread *sth, struct sender_state *s,

742 bool stream_sender_receive_data(struct stream_thread *sth, struct sender_state *s, usec_t now_ut, bool process_opcodes) {

743 EVLOOP_STATUS status = EVLOOP_STATUS_CONTINUE;

744 while(status == EVLOOP_STATUS_CONTINUE) {

727 - ssize_t rc = nd_sock_revc_nowait(&s->sock, s->rbuf.b + s->rbuf.read_len, sizeof(s->rbuf.b) - s->rbuf.read_len - 1);

745 + ssize_t rc = nd_sock_revc_nowait(&s->sock, s->thread.rbuf.b + s->thread.rbuf.read_len, s->thread.rbuf.size - s->thread.rbuf.read_len - 1);

746 if (likely(rc > 0)) {

729 - s->rbuf.read_len += rc;

747 + s->thread.rbuf.read_len += rc;

748

749 s->thread.last_traffic_ut = now_ut;

750 sth->snd.bytes_received += rc;

src/streaming/stream-thread.c

+13 -5

@@ -337,6 +337,7 @@ static void stream_thread_messages_resize(struct stream_thread *sth) {

337

338 // --------------------------------------------------------------------------------------------------------------------

339

340 +ALWAYS_INLINE_HOT_FLATTEN

341 static bool stream_thread_process_poll_slot(struct stream_thread *sth, nd_poll_result_t *ev, usec_t now_ut, size_t *replay_entries) {

342 internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );

343

@@ -520,9 +521,8 @@ void *stream_thread(void *ptr) {

521

522 rrd_collector_started();

523

524 + usec_t now_ut = now_monotonic_usec();

525 while(!exit_thread && !nd_thread_signaled_to_cancel() && service_running(SERVICE_STREAMING)) {

524 - usec_t now_ut = now_monotonic_usec();

525 -

526 if(now_ut - last_dequeue_ut >= 100 * USEC_PER_MS) {

527 last_dequeue_ut = now_ut;

528

@@ -541,6 +541,8 @@ void *stream_thread(void *ptr) {

541 receivers_waiting = sth->queue.receivers_waiting;

542 spinlock_unlock(&sth->queue.spinlock);

543

544 + // process any opcodes waiting

545 + stream_thread_process_opcodes(sth, NULL);

546

547 if(now_ut - last_check_all_nodes_ut >= nd_profile.update_every * USEC_PER_SEC) {

548 last_check_all_nodes_ut = now_ut;

@@ -583,9 +585,11 @@ void *stream_thread(void *ptr) {

585

586 worker_is_busy(WORKER_STREAM_JOB_PREP);

587

586 - if (poll_rc == 0)

588 + if (unlikely(poll_rc == 0)) {

589 // nd_poll() timed out - just loop again

590 + now_ut = now_monotonic_usec();

591 continue;

592 + }

593

594 if(unlikely(poll_rc == -1)) {

595 // nd_poll() returned an error

@@ -593,19 +597,23 @@ void *stream_thread(void *ptr) {

597 worker_is_busy(WORKER_STREAM_JOB_POLL_ERROR);

598 nd_log_limit_static_thread_var(erl, 1, 1 * USEC_PER_MS);

599 nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, "STREAM THREAD[%zu] nd_poll() returned error", sth->id);

600 + now_ut = now_monotonic_usec();

601 continue;

602 }

603

599 - if(nd_thread_signaled_to_cancel() || !service_running(SERVICE_STREAMING))

604 + if(unlikely(nd_thread_signaled_to_cancel() || !service_running(SERVICE_STREAMING)))

605 break;

606

607 // nd_poll() may have received events for a socket we have already removed

608 // so, if we don't find it in our meta index, do not access it - it has been removed

604 - if(META_GET(&sth->run.meta, (Word_t)ev.data) != ev.data)

609 + if(unlikely(META_GET(&sth->run.meta, (Word_t)ev.data) != ev.data)) {

610 + now_ut = now_monotonic_usec();

611 continue;

612 + }

613

614 now_ut = now_monotonic_usec();

615 exit_thread = stream_thread_process_poll_slot(sth, &ev, now_ut, &replay_entries);

616 + now_ut = now_monotonic_usec();

617 }

618

619 // dequeue

src/web/api/queries/query.c

+5 -3

@@ -703,7 +703,8 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_TIME_GROUPING group_method)

703 }

704 }

705

706 -static ALWAYS_INLINE void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_TIME_GROUPING add_flush) {

706 +ALWAYS_INLINE_HOT_FLATTEN

707 +static void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_TIME_GROUPING add_flush) {

708 switch(add_flush) {

709 case RRDR_GROUPING_AVERAGE:

710 tg_average_add(r, value);

@@ -760,7 +761,8 @@ static ALWAYS_INLINE void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const

761 }

762 }

763

763 -static ALWAYS_INLINE NETDATA_DOUBLE time_grouping_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr, const RRDR_TIME_GROUPING add_flush) {

764 +ALWAYS_INLINE_HOT_FLATTEN

765 +static NETDATA_DOUBLE time_grouping_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr, const RRDR_TIME_GROUPING add_flush) {

766 switch(add_flush) {

767 case RRDR_GROUPING_AVERAGE:

768 return tg_average_flush(r, rrdr_value_options_ptr);

@@ -1577,7 +1579,7 @@ static QUERY_ENGINE_OPS *rrd2rrdr_query_ops_prep(RRDR *r, size_t query_metric_id

1579 return ops;

1580 }

1581

1580 -static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_OPS *ops) {

1582 +NOT_INLINE_HOT static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_OPS *ops) {

1583 QUERY_TARGET *qt = r->internal.qt;

1584 QUERY_METRIC *qm = ops->qm;

1585