@cryptotaxi247 / netdata-1 / commits / a082cb3e8

LTO Benchmark (#19488)

* use ACQUIRE/RELEASE atomic operations in gorilla * add __attribute__((always_inline)) to functions that should always be inlined, to optimize query performance * make gorilla_reader_read inline * avoid recursion in gorilla_reader_read() * inline gorilla_writer_write() * re-arrange gorilla to be first in all case statements and mark utility functions as always inline * inline errno_clear() * inline gettid_cached() * inline rrdeng_store_metric_next() * inline aral utility functions * inline dictionary utility functions * inline pack_storage_number() * do not ask for mallinfo or malloc_info when extended pulse statistics are not set * split pulse into multiple threads: workers, memory_extended; prefer mallinfo2 over malloc_info

Costa Tsaousis committed Jan 26, 2025 at 22:24 UTC a082cb3e8f3132296a701517843b86cfb50187a1
20 files changed +486 -367
src/daemon/pulse/pulse-daemon-memory-system.c
+56 -53
@@ -109,11 +109,65 @@ cleanup:
109 }
110 #endif // HAVE_C_MALLOC_INFO
111
112 -void pulse_daemon_memory_system_do(bool extended __maybe_unused) {
112 +void pulse_daemon_memory_system_do(bool extended) {
113 + if(!extended) return;
114 +
115 + size_t glibc_mmaps = 0;
116 + bool have_mallinfo = false;
117 +
118 +#ifdef HAVE_C_MALLINFO2
119 + struct mallinfo2 mi = mallinfo2();
120 + glibc_mmaps = mi.hblks;
121 + if(mi.hblkhd || mi.fordblks) {
122 + static RRDSET *st_mallinfo = NULL;
123 + static RRDDIM *rd_used_mmap = NULL;
124 + static RRDDIM *rd_used_arena = NULL;
125 + static RRDDIM *rd_unused_fragments = NULL;
126 + static RRDDIM *rd_unused_releasable = NULL;
127 +
128 + if (unlikely(!st_mallinfo)) {
129 + st_mallinfo = rrdset_create_localhost(
130 + "netdata",
131 + "glibc_mallinfo2",
132 + NULL,
133 + "Memory Usage",
134 + NULL,
135 + "Glibc Mallinfo2 Memory Distribution",
136 + "bytes",
137 + "netdata",
138 + "pulse",
139 + 130130,
140 + localhost->rrd_update_every,
141 + RRDSET_TYPE_STACKED);
142 +
143 + rd_unused_releasable = rrddim_add(st_mallinfo, "unused releasable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
144 + rd_unused_fragments = rrddim_add(st_mallinfo, "unused fragments", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
145 + rd_used_arena = rrddim_add(st_mallinfo, "used arena", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
146 + rd_used_mmap = rrddim_add(st_mallinfo, "used mmap", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
147 + }
148 +
149 + // size_t total = mi.uordblks;
150 + size_t used_mmap = mi.hblkhd;
151 + size_t used_arena = (mi.arena > mi.fordblks) ? mi.arena - mi.fordblks : 0;
152 +
153 + size_t unused_total = mi.fordblks;
154 + size_t unused_releasable = mi.keepcost;
155 + // size_t unused_fast = mi.fsmblks;
156 + size_t unused_fragments = (unused_total > unused_releasable) ? unused_total - unused_releasable : 0;
157 +
158 + rrddim_set_by_pointer(st_mallinfo, rd_unused_releasable, (collected_number)unused_releasable);
159 + rrddim_set_by_pointer(st_mallinfo, rd_unused_fragments, (collected_number)unused_fragments);
160 + rrddim_set_by_pointer(st_mallinfo, rd_used_arena, (collected_number)used_arena);
161 + rrddim_set_by_pointer(st_mallinfo, rd_used_mmap, (collected_number)used_mmap);
162 +
163 + rrdset_done(st_mallinfo);
164 + have_mallinfo = true;
165 + }
166 +#endif // HAVE_C_MALLINFO2
167
168 #ifdef HAVE_C_MALLOC_INFO
169 size_t glibc_arenas, glibc_allocated_arenas, glibc_unused_fast, glibc_unused_rest, glibc_allocated_mmap;
116 - if(extended && parse_malloc_info(&glibc_arenas, &glibc_allocated_arenas, &glibc_unused_fast, &glibc_unused_rest, &glibc_allocated_mmap)) {
170 + if(!have_mallinfo && parse_malloc_info(&glibc_arenas, &glibc_allocated_arenas, &glibc_unused_fast, &glibc_unused_rest, &glibc_allocated_mmap)) {
171 if (glibc_arenas) {
172 static RRDSET *st_arenas = NULL;
173 static RRDDIM *rd_arenas = NULL;
@@ -180,57 +234,6 @@ void pulse_daemon_memory_system_do(bool extended __maybe_unused) {
234 }
235 #endif
236
183 - size_t glibc_mmaps = 0;
184 -
185 -#ifdef HAVE_C_MALLINFO2
186 - struct mallinfo2 mi = mallinfo2();
187 - glibc_mmaps = mi.hblks;
188 - if(extended && (mi.hblkhd || mi.fordblks)) {
189 - static RRDSET *st_mallinfo = NULL;
190 - static RRDDIM *rd_used_mmap = NULL;
191 - static RRDDIM *rd_used_arena = NULL;
192 - static RRDDIM *rd_unused_fragments = NULL;
193 - static RRDDIM *rd_unused_releasable = NULL;
194 -
195 - if (unlikely(!st_mallinfo)) {
196 - st_mallinfo = rrdset_create_localhost(
197 - "netdata",
198 - "glibc_mallinfo2",
199 - NULL,
200 - "Memory Usage",
201 - NULL,
202 - "Glibc Mallinfo2 Memory Distribution",
203 - "bytes",
204 - "netdata",
205 - "pulse",
206 - 130130,
207 - localhost->rrd_update_every,
208 - RRDSET_TYPE_STACKED);
209 -
210 - rd_unused_releasable = rrddim_add(st_mallinfo, "unused releasable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
211 - rd_unused_fragments = rrddim_add(st_mallinfo, "unused fragments", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
212 - rd_used_arena = rrddim_add(st_mallinfo, "used arena", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
213 - rd_used_mmap = rrddim_add(st_mallinfo, "used mmap", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
214 - }
215 -
216 - // size_t total = mi.uordblks;
217 - size_t used_mmap = mi.hblkhd;
218 - size_t used_arena = (mi.arena > mi.fordblks) ? mi.arena - mi.fordblks : 0;
219 -
220 - size_t unused_total = mi.fordblks;
221 - size_t unused_releasable = mi.keepcost;
222 - // size_t unused_fast = mi.fsmblks;
223 - size_t unused_fragments = (unused_total > unused_releasable) ? unused_total - unused_releasable : 0;
224 -
225 - rrddim_set_by_pointer(st_mallinfo, rd_unused_releasable, (collected_number)unused_releasable);
226 - rrddim_set_by_pointer(st_mallinfo, rd_unused_fragments, (collected_number)unused_fragments);
227 - rrddim_set_by_pointer(st_mallinfo, rd_used_arena, (collected_number)used_arena);
228 - rrddim_set_by_pointer(st_mallinfo, rd_used_mmap, (collected_number)used_mmap);
229 -
230 - rrdset_done(st_mallinfo);
231 - }
232 -#endif // HAVE_C_MALLINFO2
233 -
237 size_t netdata_mmaps = __atomic_load_n(&nd_mmap_count, __ATOMIC_RELAXED);
238 size_t total_mmaps = netdata_mmaps + glibc_mmaps;
239 {
src/daemon/pulse/pulse-daemon-memory.c
+1 -4
@@ -19,9 +19,8 @@ void rrd_slot_memory_removed(size_t added) {
19
20 struct netdata_buffers_statistics netdata_buffers_statistics = { 0 };
21
22 -void pulse_daemon_memory_system_do(bool extended);
22 +void pulse_daemon_memory_do(bool extended __maybe_unused) {
23
24 -void pulse_daemon_memory_do(bool extended) {
24 {
25 static RRDSET *st_memory = NULL;
26 static RRDDIM *rd_db_dbengine = NULL;
@@ -307,6 +306,4 @@ void pulse_daemon_memory_do(bool extended) {
306 }
307
308 // ----------------------------------------------------------------------------------------------------------------
310 -
311 - pulse_daemon_memory_system_do(extended);
309 }
src/daemon/pulse/pulse-daemon-memory.h
+1
@@ -24,6 +24,7 @@ extern struct netdata_buffers_statistics {
24
25 #if defined(PULSE_INTERNALS)
26 void pulse_daemon_memory_do(bool extended);
27 +void pulse_daemon_memory_system_do(bool extended);
28 #endif
29
30 void rrd_slot_memory_added(size_t added);
src/daemon/pulse/pulse.c
+105 -33
@@ -20,6 +20,7 @@
20 #define WORKER_JOB_ARAL 14
21 #define WORKER_JOB_NETWORK 15
22 #define WORKER_JOB_PARENTS 16
23 +#define WORKER_JOB_MEMORY_EXTENDED 17
24
25 #if WORKER_UTILIZATION_MAX_JOB_TYPES < 17
26 #error "WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 14"
@@ -43,28 +44,16 @@ static void pulse_register_workers(void) {
44 worker_register_job_name(WORKER_JOB_GORILLA, "gorilla");
45 worker_register_job_name(WORKER_JOB_HEARTBEAT, "heartbeat");
46 worker_register_job_name(WORKER_JOB_WORKERS, "workers");
46 - worker_register_job_name(WORKER_JOB_MALLOC_TRACE, "malloc_trace");
47 + worker_register_job_name(WORKER_JOB_MALLOC_TRACE, "malloc trace");
48 worker_register_job_name(WORKER_JOB_REGISTRY, "registry");
49 worker_register_job_name(WORKER_JOB_ARAL, "aral");
50 worker_register_job_name(WORKER_JOB_NETWORK, "network");
51 worker_register_job_name(WORKER_JOB_PARENTS, "parents");
51 -}
52 -
53 -static void pulse_cleanup(void *pptr)
54 -{
55 - struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr);
56 - if(!static_thread) return;
57 -
58 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
59 -
60 - pulse_workers_cleanup();
61 - worker_unregister();
62 -
63 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
52 + worker_register_job_name(WORKER_JOB_MEMORY_EXTENDED, "memory extended");
53 }
54
55 void *pulse_thread_main(void *ptr) {
67 - CLEANUP_FUNCTION_REGISTER(pulse_cleanup) cleanup_ptr = ptr;
56 + struct netdata_static_thread *static_thread = ptr;
57 pulse_register_workers();
58
59 int update_every =
@@ -135,14 +124,6 @@ void *pulse_thread_main(void *ptr) {
124 pulse_dictionary_do(pulse_extended_enabled);
125 #endif
126
138 -#ifdef NETDATA_TRACE_ALLOCATIONS
139 - worker_is_busy(WORKER_JOB_MALLOC_TRACE);
140 - pulse_trace_allocations_do(pulse_extended_enabled);
141 -#endif
142 -
143 - worker_is_busy(WORKER_JOB_WORKERS);
144 - pulse_workers_do(pulse_extended_enabled);
145 -
127 worker_is_busy(WORKER_JOB_ARAL);
128 pulse_aral_do(pulse_extended_enabled);
129
@@ -155,27 +136,105 @@ void *pulse_thread_main(void *ptr) {
136 pulse_daemon_do(pulse_extended_enabled);
137 }
138
139 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
140 + worker_unregister();
141 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
142 +
143 return NULL;
144 }
145
146 // ---------------------------------------------------------------------------------------------------------------------
147 // pulse sqlite3 thread
148
164 -static void pulse_thread_sqlite3_cleanup(void *pptr)
165 -{
166 - struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr);
167 - if (!static_thread)
168 - return;
149 +void *pulse_thread_sqlite3_main(void *ptr) {
150 + struct netdata_static_thread *static_thread = ptr;
151 + pulse_register_workers();
152
170 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
153 + int update_every =
154 + (int)config_get_duration_seconds(CONFIG_SECTION_PULSE, "update every", localhost->rrd_update_every);
155 + if (update_every < localhost->rrd_update_every) {
156 + update_every = localhost->rrd_update_every;
157 + config_set_duration_seconds(CONFIG_SECTION_PULSE, "update every", update_every);
158 + }
159 +
160 + usec_t step = update_every * USEC_PER_SEC;
161 + heartbeat_t hb;
162 + heartbeat_init(&hb, USEC_PER_SEC);
163 + usec_t real_step = USEC_PER_SEC;
164 +
165 + // keep the randomness at zero
166 + // to make sure we are not close to any other thread
167 + hb.randomness = 0;
168 +
169 + while (service_running(SERVICE_COLLECTORS)) {
170 + worker_is_idle();
171 + heartbeat_next(&hb);
172 + if (real_step < step) {
173 + real_step += USEC_PER_SEC;
174 + continue;
175 + }
176 + real_step = USEC_PER_SEC;
177 +
178 + worker_is_busy(WORKER_JOB_SQLITE3);
179 + pulse_sqlite3_do(pulse_extended_enabled);
180 + }
181
182 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
183 worker_unregister();
184 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
185 +
186 + return NULL;
187 +}
188
189 +// ---------------------------------------------------------------------------------------------------------------------
190 +// pulse workers thread
191 +
192 +void *pulse_thread_workers_main(void *ptr) {
193 + struct netdata_static_thread *static_thread = ptr;
194 + pulse_register_workers();
195 +
196 + int update_every =
197 + (int)config_get_duration_seconds(CONFIG_SECTION_PULSE, "update every", localhost->rrd_update_every);
198 + if (update_every < localhost->rrd_update_every) {
199 + update_every = localhost->rrd_update_every;
200 + config_set_duration_seconds(CONFIG_SECTION_PULSE, "update every", update_every);
201 + }
202 +
203 + usec_t step = update_every * USEC_PER_SEC;
204 + heartbeat_t hb;
205 + heartbeat_init(&hb, USEC_PER_SEC);
206 + usec_t real_step = USEC_PER_SEC;
207 +
208 + // keep the randomness at zero
209 + // to make sure we are not close to any other thread
210 + hb.randomness = 0;
211 +
212 + while (service_running(SERVICE_COLLECTORS)) {
213 + worker_is_idle();
214 + heartbeat_next(&hb);
215 + if (real_step < step) {
216 + real_step += USEC_PER_SEC;
217 + continue;
218 + }
219 + real_step = USEC_PER_SEC;
220 +
221 + worker_is_busy(WORKER_JOB_WORKERS);
222 + pulse_workers_do(pulse_extended_enabled);
223 + }
224 +
225 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
226 + pulse_workers_cleanup();
227 + worker_unregister();
228 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
229 +
230 + return NULL;
231 }
232
177 -void *pulse_thread_sqlite3_main(void *ptr) {
178 - CLEANUP_FUNCTION_REGISTER(pulse_thread_sqlite3_cleanup) cleanup_ptr = ptr;
233 +// ---------------------------------------------------------------------------------------------------------------------
234 +// pulse workers thread
235 +
236 +void *pulse_thread_memory_extended_main(void *ptr) {
237 + struct netdata_static_thread *static_thread = ptr;
238 pulse_register_workers();
239
240 int update_every =
@@ -190,6 +249,10 @@ void *pulse_thread_sqlite3_main(void *ptr) {
249 heartbeat_init(&hb, USEC_PER_SEC);
250 usec_t real_step = USEC_PER_SEC;
251
252 + // keep the randomness at zero
253 + // to make sure we are not close to any other thread
254 + hb.randomness = 0;
255 +
256 while (service_running(SERVICE_COLLECTORS)) {
257 worker_is_idle();
258 heartbeat_next(&hb);
@@ -199,9 +262,18 @@ void *pulse_thread_sqlite3_main(void *ptr) {
262 }
263 real_step = USEC_PER_SEC;
264
202 - worker_is_busy(WORKER_JOB_SQLITE3);
203 - pulse_sqlite3_do(pulse_extended_enabled);
265 +#ifdef NETDATA_TRACE_ALLOCATIONS
266 + worker_is_busy(WORKER_JOB_MALLOC_TRACE);
267 + pulse_trace_allocations_do(pulse_extended_enabled);
268 +#endif
269 +
270 + worker_is_busy(WORKER_JOB_MEMORY_EXTENDED);
271 + pulse_daemon_memory_system_do(pulse_extended_enabled);
272 }
273
274 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
275 + worker_unregister();
276 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
277 +
278 return NULL;
279 }
src/daemon/pulse/pulse.h
+2
@@ -30,6 +30,8 @@ extern bool pulse_extended_enabled;
30
31 void *pulse_thread_main(void *ptr);
32 void *pulse_thread_sqlite3_main(void *ptr);
33 +void *pulse_thread_workers_main(void *ptr);
34 +void *pulse_thread_memory_extended_main(void *ptr);
35
36 #define p1_add_fetch(variable, value) __atomic_add_fetch(variable, value, __ATOMIC_RELAXED)
37 #define p1_sub_fetch(variable, value) __atomic_sub_fetch(variable, value, __ATOMIC_RELAXED)
src/daemon/static_threads.c
+21
@@ -65,6 +65,27 @@ const struct netdata_static_thread static_threads_common[] = {
65 .init_routine = NULL,
66 .start_routine = pulse_thread_sqlite3_main
67 },
68 + {
69 + .name = "PULSE-WORKERS",
70 + .config_section = CONFIG_SECTION_PULSE,
71 + .config_name = "extended",
72 + .env_name = NULL,
73 + .global_variable = &pulse_extended_enabled,
74 + .enabled = 0, // the default value - it uses netdata.conf for users to enable it
75 + .thread = NULL,
76 + .init_routine = NULL,
77 + .start_routine = pulse_thread_workers_main
78 + },
79 + {
80 + .name = "PULSE-MEMORY",
81 + .config_section = CONFIG_SECTION_PULSE,
82 + .config_name = "extended",
83 + .env_name = NULL,
84 + .global_variable = &pulse_extended_enabled,
85 + .enabled = 0, // the default value - it uses netdata.conf for users to enable it
86 + .thread = NULL,
87 + .init_routine = NULL,
88 + .start_routine = pulse_thread_memory_extended_main},
89 {
90 .name = "PLUGINSD",
91 .config_section = NULL,
src/database/engine/page.c
+123 -113
@@ -260,17 +260,17 @@ static ARAL *pgd_get_aral_by_size_and_partition(size_t size, size_t partition) {
260 return ar;
261 }
262
263 -static inline gorilla_writer_t *pgd_gorilla_writer_alloc(size_t partition) {
263 +static ALWAYS_INLINE gorilla_writer_t *pgd_gorilla_writer_alloc(size_t partition) {
264 internal_fatal(partition >= pgd_alloc_globals.partitions, "invalid gorilla writer partition %zu", partition);
265 return aral_mallocz_marked(pgd_alloc_globals.aral_gorilla_writer[partition]);
266 }
267
268 -static inline gorilla_buffer_t *pgd_gorilla_buffer_alloc(size_t partition) {
268 +static ALWAYS_INLINE gorilla_buffer_t *pgd_gorilla_buffer_alloc(size_t partition) {
269 internal_fatal(partition >= pgd_alloc_globals.partitions, "invalid gorilla buffer partition %zu", partition);
270 return aral_mallocz_marked(pgd_alloc_globals.aral_gorilla_buffer[partition]);
271 }
272
273 -static inline PGD *pgd_alloc(bool for_collector) {
273 +static ALWAYS_INLINE PGD *pgd_alloc(bool for_collector) {
274 size_t partition = gettid_cached() % pgd_alloc_globals.partitions;
275 PGD *pgd;
276
@@ -283,7 +283,7 @@ static inline PGD *pgd_alloc(bool for_collector) {
283 return pgd;
284 }
285
286 -static inline void *pgd_data_alloc(size_t size, size_t partition, bool for_collector) {
286 +static ALWAYS_INLINE void *pgd_data_alloc(size_t size, size_t partition, bool for_collector) {
287 ARAL *ar = pgd_get_aral_by_size_and_partition(size, partition);
288 if(ar) {
289 int64_t padding = (int64_t)aral_requested_element_size(ar) - (int64_t)size;
@@ -298,7 +298,7 @@ static inline void *pgd_data_alloc(size_t size, size_t partition, bool for_colle
298 return mallocz(size);
299 }
300
301 -static void pgd_data_free(void *page, size_t size, size_t partition) {
301 +static ALWAYS_INLINE void pgd_data_free(void *page, size_t size, size_t partition) {
302 ARAL *ar = pgd_get_aral_by_size_and_partition(size, partition);
303 if(ar) {
304 int64_t padding = (int64_t)aral_requested_element_size(ar) - (int64_t)size;
@@ -311,7 +311,7 @@ static void pgd_data_free(void *page, size_t size, size_t partition) {
311 timing_dbengine_evict_step(TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_TIER1_ARAL);
312 }
313
314 -static void pgd_data_unmark(void *page, size_t size, size_t partition) {
314 +static ALWAYS_INLINE void pgd_data_unmark(void *page, size_t size, size_t partition) {
315 if(!page) return;
316
317 ARAL *ar = pgd_get_aral_by_size_and_partition(size, partition);
@@ -329,11 +329,11 @@ static size_t pgd_data_footprint(size_t size, size_t partition) {
329
330 // ----------------------------------------------------------------------------
331
332 -void *dbengine_extent_alloc(size_t size) {
332 +ALWAYS_INLINE void *dbengine_extent_alloc(size_t size) {
333 return pgd_data_alloc(size, 0, false);
334 }
335
336 -void dbengine_extent_free(void *extent, size_t size) {
336 +ALWAYS_INLINE void dbengine_extent_free(void *extent, size_t size) {
337 pgd_data_free(extent, size, 0);
338 }
339
@@ -351,17 +351,6 @@ PGD *pgd_create(uint8_t type, uint32_t slots) {
351 pg->slots = slots;
352
353 switch (type) {
354 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
355 - case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
356 - uint32_t size = slots * page_type_size[type];
357 -
358 - internal_fatal(!size || slots == 1,
359 - "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
360 -
361 - pg->raw.size = size;
362 - pg->raw.data = pgd_data_alloc(size, pg->partition, true);
363 - break;
364 - }
354 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
355 internal_fatal(slots == 1,
356 "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
@@ -379,6 +368,19 @@ PGD *pgd_create(uint8_t type, uint32_t slots) {
368
369 break;
370 }
371 +
372 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
373 + case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
374 + uint32_t size = slots * page_type_size[type];
375 +
376 + internal_fatal(!size || slots == 1,
377 + "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
378 +
379 + pg->raw.size = size;
380 + pg->raw.data = pgd_data_alloc(size, pg->partition, true);
381 + break;
382 + }
383 +
384 default:
385 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, type);
386 aral_freez(pgd_alloc_globals.aral_pgd[pg->partition], pg);
@@ -401,16 +403,6 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size) {
403
404 switch (type)
405 {
404 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
405 - case RRDENG_PAGE_TYPE_ARRAY_TIER1:
406 - pg->used = size / page_type_size[type];
407 - pg->slots = pg->used;
408 -
409 - pg->raw.size = size;
410 - pg->raw.data = pgd_data_alloc(size, pg->partition, false);
411 - memcpy(pg->raw.data, base, size);
412 - break;
413 -
406 case RRDENG_PAGE_TYPE_GORILLA_32BIT:
407 internal_fatal(size == 0, "Asked to create page with 0 data!!!");
408 internal_fatal(size % sizeof(uint32_t), "Unaligned gorilla buffer size");
@@ -427,6 +419,16 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size) {
419 pg->slots = pg->used;
420 break;
421
422 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
423 + case RRDENG_PAGE_TYPE_ARRAY_TIER1:
424 + pg->used = size / page_type_size[type];
425 + pg->slots = pg->used;
426 +
427 + pg->raw.size = size;
428 + pg->raw.data = pgd_data_alloc(size, pg->partition, false);
429 + memcpy(pg->raw.data, base, size);
430 + break;
431 +
432 default:
433 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, type);
434 aral_freez(pgd_alloc_globals.aral_pgd[pg->partition], pg);
@@ -446,11 +448,6 @@ void pgd_free(PGD *pg) {
448
449 switch (pg->type)
450 {
449 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
450 - case RRDENG_PAGE_TYPE_ARRAY_TIER1:
451 - pgd_data_free(pg->raw.data, pg->raw.size, pg->partition);
452 - break;
453 -
451 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
452 if (pg->states & PGD_STATE_CREATED_FROM_DISK)
453 {
@@ -498,6 +495,12 @@ void pgd_free(PGD *pg) {
495
496 break;
497 }
498 +
499 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
500 + case RRDENG_PAGE_TYPE_ARRAY_TIER1:
501 + pgd_data_free(pg->raw.data, pg->raw.size, pg->partition);
502 + break;
503 +
504 default:
505 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type);
506 break;
@@ -522,11 +525,6 @@ static void pgd_aral_unmark(PGD *pg) {
525
526 switch (pg->type)
527 {
525 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
526 - case RRDENG_PAGE_TYPE_ARRAY_TIER1:
527 - pgd_data_unmark(pg->raw.data, pg->raw.size, pg->partition);
528 - break;
529 -
528 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
529 if (pg->states & PGD_STATE_CREATED_FROM_DISK)
530 pgd_data_unmark(pg->raw.data, pg->raw.size, pg->partition);
@@ -550,6 +548,12 @@ static void pgd_aral_unmark(PGD *pg) {
548
549 break;
550 }
551 +
552 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
553 + case RRDENG_PAGE_TYPE_ARRAY_TIER1:
554 + pgd_data_unmark(pg->raw.data, pg->raw.size, pg->partition);
555 + break;
556 +
557 default:
558 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type);
559 break;
@@ -564,12 +568,12 @@ static void pgd_aral_unmark(PGD *pg) {
568 // ----------------------------------------------------------------------------
569 // utility functions
570
567 -uint32_t pgd_type(PGD *pg)
571 +ALWAYS_INLINE uint32_t pgd_type(PGD *pg)
572 {
573 return pg->type;
574 }
575
572 -bool pgd_is_empty(PGD *pg)
576 +ALWAYS_INLINE bool pgd_is_empty(PGD *pg)
577 {
578 if (!pg)
579 return true;
@@ -586,7 +590,7 @@ bool pgd_is_empty(PGD *pg)
590 return false;
591 }
592
589 -uint32_t pgd_slots_used(PGD *pg)
593 +ALWAYS_INLINE uint32_t pgd_slots_used(PGD *pg)
594 {
595 if (!pg)
596 return 0;
@@ -597,7 +601,7 @@ uint32_t pgd_slots_used(PGD *pg)
601 return pg->used;
602 }
603
600 -uint32_t pgd_capacity(PGD *pg) {
604 +ALWAYS_INLINE uint32_t pgd_capacity(PGD *pg) {
605 if (!pg)
606 return 0;
607
@@ -619,11 +623,6 @@ uint32_t pgd_memory_footprint(PGD *pg)
623 size_t footprint = pgd_alloc_globals.sizeof_pgd;
624
625 switch (pg->type) {
622 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
623 - case RRDENG_PAGE_TYPE_ARRAY_TIER1:
624 - footprint += pgd_data_footprint(pg->raw.size, pg->partition);
625 - break;
626 -
626 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
627 if (pg->states & PGD_STATE_CREATED_FROM_DISK)
628 footprint += pgd_data_footprint(pg->raw.size, pg->partition);
@@ -635,6 +634,11 @@ uint32_t pgd_memory_footprint(PGD *pg)
634 break;
635 }
636
637 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
638 + case RRDENG_PAGE_TYPE_ARRAY_TIER1:
639 + footprint += pgd_data_footprint(pg->raw.size, pg->partition);
640 + break;
641 +
642 default:
643 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type);
644 break;
@@ -655,11 +659,6 @@ uint32_t pgd_buffer_memory_footprint(PGD *pg)
659 size_t footprint = 0;
660
661 switch (pg->type) {
658 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
659 - case RRDENG_PAGE_TYPE_ARRAY_TIER1:
660 - footprint = pg->raw.size;
661 - break;
662 -
662 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
663 if (pg->states & PGD_STATE_CREATED_FROM_DISK)
664 footprint = pg->raw.size;
@@ -669,6 +668,11 @@ uint32_t pgd_buffer_memory_footprint(PGD *pg)
668 break;
669 }
670
671 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
672 + case RRDENG_PAGE_TYPE_ARRAY_TIER1:
673 + footprint = pg->raw.size;
674 + break;
675 +
676 default:
677 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type);
678 break;
@@ -688,14 +692,6 @@ uint32_t pgd_disk_footprint(PGD *pg)
692 pgd_aral_unmark(pg);
693
694 switch (pg->type) {
691 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
692 - case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
693 - uint32_t used_size = pg->used * page_type_size[pg->type];
694 - internal_fatal(used_size > pg->raw.size, "Wrong disk footprint page size");
695 - size = used_size;
696 -
697 - break;
698 - }
695 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
696 if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR ||
697 pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING ||
@@ -723,6 +719,16 @@ uint32_t pgd_disk_footprint(PGD *pg)
719
720 break;
721 }
722 +
723 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
724 + case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
725 + uint32_t used_size = pg->used * page_type_size[pg->type];
726 + internal_fatal(used_size > pg->raw.size, "Wrong disk footprint page size");
727 + size = used_size;
728 +
729 + break;
730 + }
731 +
732 default:
733 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type);
734 break;
@@ -741,10 +747,6 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size)
747 pgd_disk_footprint(pg), dst_size);
748
749 switch (pg->type) {
744 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
745 - case RRDENG_PAGE_TYPE_ARRAY_TIER1:
746 - memcpy(dst, pg->raw.data, dst_size);
747 - break;
750 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
751 if ((pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) == 0)
752 fatal("Copying to extent is supported only for PGDs that are scheduled for flushing.");
@@ -762,6 +764,12 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size)
764 pg, pg->gorilla.writer, dst_size, pg->gorilla.num_buffers);
765 break;
766 }
767 +
768 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
769 + case RRDENG_PAGE_TYPE_ARRAY_TIER1:
770 + memcpy(dst, pg->raw.data, dst_size);
771 + break;
772 +
773 default:
774 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type);
775 break;
@@ -774,7 +782,7 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size)
782 // data collection
783
784 // returns additional memory that may have been allocated to store this point
777 -size_t pgd_append_point(PGD *pg,
785 +ALWAYS_INLINE size_t pgd_append_point(PGD *pg,
786 usec_t point_in_time_ut __maybe_unused,
787 NETDATA_DOUBLE n,
788 NETDATA_DOUBLE min_value,
@@ -799,14 +807,28 @@ size_t pgd_append_point(PGD *pg,
807 fatal("Data collection on page already scheduled for flushing");
808
809 switch (pg->type) {
802 - case RRDENG_PAGE_TYPE_ARRAY_32BIT: {
803 - storage_number *tier0_metric_data = (storage_number *)pg->raw.data;
810 + case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
811 + pg->used++;
812 storage_number t = pack_storage_number(n, flags);
805 - tier0_metric_data[pg->used++] = t;
813
814 if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && does_storage_number_exist(t))
815 pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY;
816
817 + bool ok = gorilla_writer_write(pg->gorilla.writer, t);
818 + if (!ok) {
819 + gorilla_buffer_t *new_buffer = pgd_gorilla_buffer_alloc(pg->partition);
820 + memset(new_buffer, 0, RRDENG_GORILLA_32BIT_BUFFER_SIZE);
821 +
822 + gorilla_writer_add_buffer(pg->gorilla.writer, new_buffer, RRDENG_GORILLA_32BIT_BUFFER_SLOTS);
823 + pg->gorilla.num_buffers += 1;
824 + pulse_gorilla_hot_buffer_added();
825 +
826 + ok = gorilla_writer_write(pg->gorilla.writer, t);
827 + internal_fatal(ok == false, "Failed to writer value in newly allocated gorilla buffer.");
828 +
829 + return RRDENG_GORILLA_32BIT_BUFFER_SIZE;
830 + }
831 +
832 break;
833 }
834 case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
@@ -824,28 +846,14 @@ size_t pgd_append_point(PGD *pg,
846
847 break;
848 }
827 - case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
828 - pg->used++;
849 + case RRDENG_PAGE_TYPE_ARRAY_32BIT: {
850 + storage_number *tier0_metric_data = (storage_number *)pg->raw.data;
851 storage_number t = pack_storage_number(n, flags);
852 + tier0_metric_data[pg->used++] = t;
853
854 if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && does_storage_number_exist(t))
855 pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY;
856
834 - bool ok = gorilla_writer_write(pg->gorilla.writer, t);
835 - if (!ok) {
836 - gorilla_buffer_t *new_buffer = pgd_gorilla_buffer_alloc(pg->partition);
837 - memset(new_buffer, 0, RRDENG_GORILLA_32BIT_BUFFER_SIZE);
838 -
839 - gorilla_writer_add_buffer(pg->gorilla.writer, new_buffer, RRDENG_GORILLA_32BIT_BUFFER_SLOTS);
840 - pg->gorilla.num_buffers += 1;
841 - pulse_gorilla_hot_buffer_added();
842 -
843 - ok = gorilla_writer_write(pg->gorilla.writer, t);
844 - internal_fatal(ok == false, "Failed to writer value in newly allocated gorilla buffer.");
845 -
846 - return RRDENG_GORILLA_32BIT_BUFFER_SIZE;
847 - }
848 -
857 break;
858 }
859 default:
@@ -864,10 +872,6 @@ static void pgdc_seek(PGDC *pgdc, uint32_t position)
872 PGD *pg = pgdc->pgd;
873
874 switch (pg->type) {
867 - case RRDENG_PAGE_TYPE_ARRAY_32BIT:
868 - case RRDENG_PAGE_TYPE_ARRAY_TIER1:
869 - pgdc->slots = pgdc->pgd->used;
870 - break;
875 case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
876 if (pg->states & PGD_STATE_CREATED_FROM_DISK) {
877 pgdc->slots = pgdc->pgd->slots;
@@ -900,6 +904,12 @@ static void pgdc_seek(PGDC *pgdc, uint32_t position)
904
905 break;
906 }
907 +
908 + case RRDENG_PAGE_TYPE_ARRAY_32BIT:
909 + case RRDENG_PAGE_TYPE_ARRAY_TIER1:
910 + pgdc->slots = pgdc->pgd->used;
911 + break;
912 +
913 default:
914 netdata_log_error("%s() - Unknown page type: %uc", __FUNCTION__, pg->type);
915 break;
@@ -925,7 +935,7 @@ void pgdc_reset(PGDC *pgdc, PGD *pgd, uint32_t position)
935 pgdc_seek(pgdc, position);
936 }
937
928 -bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused, STORAGE_POINT *sp)
938 +ALWAYS_INLINE bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused, STORAGE_POINT *sp)
939 {
940 if (!pgdc->pgd || pgdc->pgd == PGD_EMPTY || pgdc->position >= pgdc->slots)
941 {
@@ -937,16 +947,22 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
947
948 switch (pgdc->pgd->type)
949 {
940 - case RRDENG_PAGE_TYPE_ARRAY_32BIT: {
941 - storage_number *array = (storage_number *) pgdc->pgd->raw.data;
942 - storage_number n = array[pgdc->position++];
950 + case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
951 + pgdc->position++;
952
944 - sp->min = sp->max = sp->sum = unpack_storage_number(n);
945 - sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS);
946 - sp->count = 1;
947 - sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0;
953 + uint32_t n = 666666666;
954 + bool ok = gorilla_reader_read(&pgdc->gr, &n);
955
949 - return true;
956 + if (ok) {
957 + sp->min = sp->max = sp->sum = unpack_storage_number(n);
958 + sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS);
959 + sp->count = 1;
960 + sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0;
961 + } else {
962 + storage_point_empty(*sp, sp->start_time_s, sp->end_time_s);
963 + }
964 +
965 + return ok;
966 }
967 case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
968 storage_number_tier1_t *array = (storage_number_tier1_t *) pgdc->pgd->raw.data;
@@ -961,22 +977,16 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
977
978 return true;
979 }
964 - case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
965 - pgdc->position++;
966 -
967 - uint32_t n = 666666666;
968 - bool ok = gorilla_reader_read(&pgdc->gr, &n);
980 + case RRDENG_PAGE_TYPE_ARRAY_32BIT: {
981 + storage_number *array = (storage_number *) pgdc->pgd->raw.data;
982 + storage_number n = array[pgdc->position++];
983
970 - if (ok) {
971 - sp->min = sp->max = sp->sum = unpack_storage_number(n);
972 - sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS);
973 - sp->count = 1;
974 - sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0;
975 - } else {
976 - storage_point_empty(*sp, sp->start_time_s, sp->end_time_s);
977 - }
984 + sp->min = sp->max = sp->sum = unpack_storage_number(n);
985 + sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS);
986 + sp->count = 1;
987 + sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0;
988
979 - return ok;
989 + return true;
990 }
991 default: {
992 static bool logged = false;
src/database/engine/rrdengineapi.c
+4 -4
@@ -487,7 +487,7 @@ static PGD *rrdeng_alloc_new_page_data(struct rrdeng_collect_handle *handle, use
487 return d;
488 }
489
490 -static void rrdeng_store_metric_append_point(STORAGE_COLLECT_HANDLE *sch,
490 +static ALWAYS_INLINE void rrdeng_store_metric_append_point(STORAGE_COLLECT_HANDLE *sch,
491 const usec_t point_in_time_ut,
492 const NETDATA_DOUBLE n,
493 const NETDATA_DOUBLE min_value,
@@ -570,7 +570,7 @@ static void store_metric_next_error_log(struct rrdeng_collect_handle *handle __m
570 #endif
571 }
572
573 -void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *sch,
573 +ALWAYS_INLINE void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *sch,
574 const usec_t point_in_time_ut,
575 const NETDATA_DOUBLE n,
576 const NETDATA_DOUBLE min_value,
@@ -803,7 +803,7 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *smh,
803 }
804 }
805
806 -static bool rrdeng_load_page_next(struct storage_engine_query_handle *seqh, bool debug_this __maybe_unused) {
806 +static inline bool rrdeng_load_page_next(struct storage_engine_query_handle *seqh, bool debug_this __maybe_unused) {
807 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)seqh->handle;
808 struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);
809
@@ -874,7 +874,7 @@ static bool rrdeng_load_page_next(struct storage_engine_query_handle *seqh, bool
874 // Returns the metric and sets its timestamp into current_time
875 // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
876 // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
877 -STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh) {
877 +ALWAYS_INLINE STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh) {
878 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)seqh->handle;
879 STORAGE_POINT sp;
880
src/database/ram/rrddim_mem.c
+1 -1
@@ -390,7 +390,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_h
390 // Returns the metric and sets its timestamp into current_time
391 // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
392 // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
393 -STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh) {
393 +ALWAYS_INLINE STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh) {
394 struct mem_query_handle* h = (struct mem_query_handle*)seqh->handle;
395 struct mem_metric_handle *mh = (struct mem_metric_handle *)h->smh;
396 RRDDIM *rd = mh->rd;
src/database/storage-engine.h
+2 -2
@@ -324,7 +324,7 @@ static inline void storage_engine_query_init(
324
325 STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *seqh);
326 STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *seqh);
327 -static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *seqh) {
327 +static ALWAYS_INLINE STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *seqh) {
328 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
329
330 #ifdef ENABLE_DBENGINE
@@ -336,7 +336,7 @@ static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engi
336
337 int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *seqh);
338 int rrddim_query_is_finished(struct storage_engine_query_handle *seqh);
339 -static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *seqh) {
339 +static ALWAYS_INLINE int storage_engine_query_is_finished(struct storage_engine_query_handle *seqh) {
340 internal_fatal(!is_valid_backend(seqh->seb), "STORAGE: invalid backend");
341
342 #ifdef ENABLE_DBENGINE
src/libnetdata/aral/aral.c
+21 -21
@@ -140,14 +140,14 @@ const char *aral_name(ARAL *ar) {
140 return ar->config.name;
141 }
142
143 -static inline void aral_element_given(ARAL *ar, ARAL_PAGE *page) {
143 +static ALWAYS_INLINE void aral_element_given(ARAL *ar, ARAL_PAGE *page) {
144 if(ar->config.mmap.enabled || page->mapped)
145 __atomic_add_fetch(&ar->stats->mmap.used_bytes, ar->config.requested_element_size, __ATOMIC_RELAXED);
146 else
147 __atomic_add_fetch(&ar->stats->malloc.used_bytes, ar->config.requested_element_size, __ATOMIC_RELAXED);
148 }
149
150 -static inline void aral_element_returned(ARAL *ar, ARAL_PAGE *page) {
150 +static ALWAYS_INLINE void aral_element_returned(ARAL *ar, ARAL_PAGE *page) {
151 if(ar->config.mmap.enabled || page->mapped)
152 __atomic_sub_fetch(&ar->stats->mmap.used_bytes, ar->config.requested_element_size, __ATOMIC_RELAXED);
153 else
@@ -207,12 +207,12 @@ struct aral_statistics *aral_get_statistics(ARAL *ar) {
207 return ar->stats;
208 }
209
210 -static inline void aral_lock_with_trace(ARAL *ar, const char *func) {
210 +static ALWAYS_INLINE void aral_lock_with_trace(ARAL *ar, const char *func) {
211 if(likely(!(ar->config.options & ARAL_LOCKLESS)))
212 spinlock_lock_with_trace(&ar->aral_lock.spinlock, func);
213 }
214
215 -static inline void aral_unlock_with_trace(ARAL *ar, const char *func) {
215 +static ALWAYS_INLINE void aral_unlock_with_trace(ARAL *ar, const char *func) {
216 if(likely(!(ar->config.options & ARAL_LOCKLESS)))
217 spinlock_unlock_with_trace(&ar->aral_lock.spinlock, func);
218 }
@@ -220,27 +220,27 @@ static inline void aral_unlock_with_trace(ARAL *ar, const char *func) {
220 #define aral_lock(ar) aral_lock_with_trace(ar, __FUNCTION__)
221 #define aral_unlock(ar) aral_unlock_with_trace(ar, __FUNCTION__)
222
223 -static inline void aral_page_available_lock(ARAL *ar, ARAL_PAGE *page) {
223 +static ALWAYS_INLINE void aral_page_available_lock(ARAL *ar, ARAL_PAGE *page) {
224 if(likely(!(ar->config.options & ARAL_LOCKLESS)))
225 spinlock_lock(&page->available.spinlock);
226 }
227
228 -static inline void aral_page_available_unlock(ARAL *ar, ARAL_PAGE *page) {
228 +static ALWAYS_INLINE void aral_page_available_unlock(ARAL *ar, ARAL_PAGE *page) {
229 if(likely(!(ar->config.options & ARAL_LOCKLESS)))
230 spinlock_unlock(&page->available.spinlock);
231 }
232
233 -static inline void aral_page_incoming_lock(ARAL *ar, ARAL_PAGE *page, size_t partition) {
233 +static ALWAYS_INLINE void aral_page_incoming_lock(ARAL *ar, ARAL_PAGE *page, size_t partition) {
234 if(likely(!(ar->config.options & ARAL_LOCKLESS)))
235 spinlock_lock(&page->incoming[partition].spinlock);
236 }
237
238 -static inline void aral_page_incoming_unlock(ARAL *ar, ARAL_PAGE *page, size_t partition) {
238 +static ALWAYS_INLINE void aral_page_incoming_unlock(ARAL *ar, ARAL_PAGE *page, size_t partition) {
239 if(likely(!(ar->config.options & ARAL_LOCKLESS)))
240 spinlock_unlock(&page->incoming[partition].spinlock);
241 }
242
243 -static inline bool aral_adders_trylock(ARAL *ar, bool marked) {
243 +static ALWAYS_INLINE bool aral_adders_trylock(ARAL *ar, bool marked) {
244 if(likely(!(ar->config.options & ARAL_LOCKLESS))) {
245 size_t idx = mark_to_idx(marked);
246 return spinlock_trylock(&ar->ops[idx].adders.spinlock);
@@ -249,14 +249,14 @@ static inline bool aral_adders_trylock(ARAL *ar, bool marked) {
249 return true;
250 }
251
252 -static inline void aral_adders_lock(ARAL *ar, bool marked) {
252 +static ALWAYS_INLINE void aral_adders_lock(ARAL *ar, bool marked) {
253 if(likely(!(ar->config.options & ARAL_LOCKLESS))) {
254 size_t idx = mark_to_idx(marked);
255 spinlock_lock(&ar->ops[idx].adders.spinlock);
256 }
257 }
258
259 -static inline void aral_adders_unlock(ARAL *ar, bool marked) {
259 +static ALWAYS_INLINE void aral_adders_unlock(ARAL *ar, bool marked) {
260 if(likely(!(ar->config.options & ARAL_LOCKLESS))) {
261 size_t idx = mark_to_idx(marked);
262 spinlock_unlock(&ar->ops[idx].adders.spinlock);
@@ -367,7 +367,7 @@ static inline ARAL_PAGE *find_page_with_allocation_internal_check(ARAL *ar, void
367 // Tagging the pointer with the 'marked' flag
368
369 // Retrieving the pointer and the 'marked' flag
370 -static ARAL_PAGE *aral_get_page_pointer_after_element___do_NOT_have_aral_lock(ARAL *ar, void *ptr, bool *marked) {
370 +static ALWAYS_INLINE ARAL_PAGE *aral_get_page_pointer_after_element___do_NOT_have_aral_lock(ARAL *ar, void *ptr, bool *marked) {
371 uint8_t *data = ptr;
372 uintptr_t *page_ptr = (uintptr_t *)&data[ar->config.element_ptr_offset];
373 uintptr_t tagged_page = __atomic_load_n(page_ptr, __ATOMIC_ACQUIRE); // Atomically load the tagged pointer
@@ -403,7 +403,7 @@ static ARAL_PAGE *aral_get_page_pointer_after_element___do_NOT_have_aral_lock(AR
403 return page;
404 }
405
406 -static void aral_set_page_pointer_after_element___do_NOT_have_aral_lock(ARAL *ar, void *page, void *ptr, bool marked) {
406 +static ALWAYS_INLINE void aral_set_page_pointer_after_element___do_NOT_have_aral_lock(ARAL *ar, void *page, void *ptr, bool marked) {
407 uint8_t *data = ptr;
408 uintptr_t *page_ptr = (uintptr_t *)&data[ar->config.element_ptr_offset];
409 uintptr_t tagged_page = (uintptr_t)page; // Cast the pointer to an integer
@@ -431,7 +431,7 @@ static inline void aral_free_validate_internal_check(ARAL *ar, ARAL_FREE *fr) {
431 // --------------------------------------------------------------------------------------------------------------------
432 // page size management
433
434 -static size_t aral_element_slot_size(size_t requested_element_size, bool usable) {
434 +static ALWAYS_INLINE size_t aral_element_slot_size(size_t requested_element_size, bool usable) {
435 // we need to add a page pointer after the element
436 // so, first align the element size to the pointer size
437 size_t element_size = memory_alignment(requested_element_size, sizeof(uintptr_t));
@@ -456,7 +456,7 @@ size_t aral_optimal_malloc_page_size(void) {
456 return ARAL_MAX_PAGE_SIZE_MALLOC;
457 }
458
459 -static size_t aral_elements_in_page_size(ARAL *ar, size_t page_size) {
459 +static ALWAYS_INLINE size_t aral_elements_in_page_size(ARAL *ar, size_t page_size) {
460 if(ar->config.mmap.enabled)
461 return page_size / ar->config.element_size;
462
@@ -465,7 +465,7 @@ static size_t aral_elements_in_page_size(ARAL *ar, size_t page_size) {
465 return remaining / ar->config.element_size;
466 }
467
468 -static size_t aral_next_allocation_size___adders_lock_needed(ARAL *ar, bool marked) {
468 +static ALWAYS_INLINE size_t aral_next_allocation_size___adders_lock_needed(ARAL *ar, bool marked) {
469 size_t idx = mark_to_idx(marked);
470 size_t size = ar->ops[idx].adders.allocation_size;
471
@@ -583,7 +583,7 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_
583 return page;
584 }
585
586 -void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
586 +static void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
587 size_t idx = mark_to_idx(page->started_marked);
588 __atomic_store_n(&ar->ops[idx].atomic.last_allocated_or_deallocated, true, __ATOMIC_RELAXED);
589
@@ -636,7 +636,7 @@ void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_
636 __atomic_sub_fetch(&ar->stats->structures.allocated_bytes, structures_size, __ATOMIC_RELAXED);
637 }
638
639 -static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
639 +static ALWAYS_INLINE ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
640 size_t idx = mark_to_idx(marked);
641 __atomic_add_fetch(&ar->ops[idx].atomic.allocators, 1, __ATOMIC_RELAXED);
642 aral_lock(ar);
@@ -757,7 +757,7 @@ static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool mar
757 return page;
758 }
759
760 -static void *aral_get_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *page, bool marked) {
760 +static ALWAYS_INLINE void *aral_get_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *page, bool marked) {
761 // Try fast path first
762 uint64_t slot = __atomic_fetch_add(&page->elements_segmented, 1, __ATOMIC_ACQUIRE);
763 if (slot < page->max_elements) {
@@ -821,7 +821,7 @@ static inline void aral_add_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *pa
821 aral_page_incoming_unlock(ar, page, partition);
822 }
823
824 -void *aral_callocz_internal(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
824 +ALWAYS_INLINE void *aral_callocz_internal(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
825 void *r = aral_mallocz_internal(ar, marked TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
826 memset(r, 0, ar->config.requested_element_size);
827 return r;
@@ -844,7 +844,7 @@ void *aral_mallocz_internal(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEF
844 }
845
846 // returns true if it moved the page to the unmarked list
847 -static ARAL_PAGE **aral_remove_marked_allocation___aral_lock_needed(ARAL *ar, ARAL_PAGE **head_ptr, ARAL_PAGE *page) {
847 +static ALWAYS_INLINE ARAL_PAGE **aral_remove_marked_allocation___aral_lock_needed(ARAL *ar, ARAL_PAGE **head_ptr, ARAL_PAGE *page) {
848 internal_fatal(!page->aral_lock.marked_elements, "marked elements refcount found zero");
849 internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");
850
src/libnetdata/common.h
+4
@@ -337,8 +337,12 @@ typedef uint32_t uid_t;
337
338 #ifdef __GNUC__
339 #define UNUSED_FUNCTION(x) __attribute__((unused)) UNUSED_##x
340 +#define ALWAYS_INLINE inline __attribute__((always_inline))
341 +#define ALWAYS_INLINE_ONLY __attribute__((always_inline))
342 #else
343 #define UNUSED_FUNCTION(x) UNUSED_##x
344 +#define ALWAYS_INLINE inline
345 +#define ALWAYS_INLINE_ONLY
346 #endif
347
348 // --------------------------------------------------------------------------------------------------------------------
src/libnetdata/dictionary/dictionary-traversal.c
+1 -1
@@ -52,7 +52,7 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
52 return dfe->value;
53 }
54
55 -void *dictionary_foreach_next(DICTFE *dfe) {
55 +ALWAYS_INLINE void *dictionary_foreach_next(DICTFE *dfe) {
56 if(unlikely(!dfe || !dfe->dict)) return NULL;
57
58 if(unlikely(is_dictionary_destroyed(dfe->dict))) {
src/libnetdata/dictionary/dictionary.c
+5 -3
@@ -120,7 +120,8 @@ size_t dictionary_version(DICTIONARY *dict) {
120
121 return __atomic_load_n(&dict->version, __ATOMIC_RELAXED);
122 }
123 -size_t dictionary_entries(DICTIONARY *dict) {
123 +
124 +ALWAYS_INLINE size_t dictionary_entries(DICTIONARY *dict) {
125 if(unlikely(!dict)) return 0;
126
127 // this is required for views to return the right number
@@ -131,6 +132,7 @@ size_t dictionary_entries(DICTIONARY *dict) {
132
133 return entries;
134 }
135 +
136 size_t dictionary_referenced_items(DICTIONARY *dict) {
137 if(unlikely(!dict)) return 0;
138
@@ -742,11 +744,11 @@ void dictionary_acquired_item_release(DICTIONARY *dict, DICT_ITEM_CONST DICTIONA
744 // ----------------------------------------------------------------------------
745 // get the name/value of an item
746
745 -const char *dictionary_acquired_item_name(DICT_ITEM_CONST DICTIONARY_ITEM *item) {
747 +ALWAYS_INLINE const char *dictionary_acquired_item_name(DICT_ITEM_CONST DICTIONARY_ITEM *item) {
748 return item_get_name(item);
749 }
750
749 -void *dictionary_acquired_item_value(DICT_ITEM_CONST DICTIONARY_ITEM *item) {
751 +ALWAYS_INLINE void *dictionary_acquired_item_value(DICT_ITEM_CONST DICTIONARY_ITEM *item) {
752 if(likely(item))
753 return item->shared->value;
754
src/libnetdata/gorilla/gorilla.cc
+133 -126
@@ -1,5 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 +#include "libnetdata/common.h"
4 #include "gorilla.h"
5
6 #include <cassert>
@@ -112,17 +113,17 @@ void gorilla_writer_add_buffer(gorilla_writer_t *gw, gorilla_buffer_t *gbuf, siz
113 if (gw->last_buffer)
114 gw->last_buffer->header.next = gbuf;
115
115 - __atomic_store_n(&gw->last_buffer, gbuf, __ATOMIC_RELAXED);
116 + __atomic_store_n(&gw->last_buffer, gbuf, __ATOMIC_RELEASE);
117 }
118
119 uint32_t gorilla_writer_entries(const gorilla_writer_t *gw) {
120 uint32_t entries = 0;
121
121 - const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_SEQ_CST);
122 + const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
123 do {
123 - const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_SEQ_CST);
124 + const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
125
125 - entries += __atomic_load_n(&curr_gbuf->header.entries, __ATOMIC_SEQ_CST);
126 + entries += __atomic_load_n(&curr_gbuf->header.entries, __ATOMIC_ACQUIRE);
127
128 curr_gbuf = next_gbuf;
129 } while (curr_gbuf);
@@ -130,66 +131,68 @@ uint32_t gorilla_writer_entries(const gorilla_writer_t *gw) {
131 return entries;
132 }
133
133 -bool gorilla_writer_write(gorilla_writer_t *gw, uint32_t number)
134 -{
135 - gorilla_header_t *hdr = &gw->last_buffer->header;
136 - uint32_t *data = gw->last_buffer->data;
134 +extern "C" {
135 + ALWAYS_INLINE_ONLY bool gorilla_writer_write(gorilla_writer_t *gw, uint32_t number)
136 + {
137 + gorilla_header_t *hdr = &gw->last_buffer->header;
138 + uint32_t *data = gw->last_buffer->data;
139 +
140 + // this is the first number we are writing
141 + if (hdr->entries == 0) {
142 + if (hdr->nbits + bit_size<uint32_t>() >= gw->capacity)
143 + return false;
144 + bit_buffer_write(data, hdr->nbits, number, bit_size<uint32_t>());
145
138 - // this is the first number we are writing
139 - if (hdr->entries == 0) {
140 - if (hdr->nbits + bit_size<uint32_t>() >= gw->capacity)
141 - return false;
142 - bit_buffer_write(data, hdr->nbits, number, bit_size<uint32_t>());
146 + __atomic_fetch_add(&hdr->nbits, bit_size<uint32_t>(), __ATOMIC_RELEASE);
147 + __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELEASE);
148 + gw->prev_number = number;
149 + return true;
150 + }
151
144 - __atomic_fetch_add(&hdr->nbits, bit_size<uint32_t>(), __ATOMIC_RELAXED);
145 - __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELAXED);
146 - gw->prev_number = number;
147 - return true;
148 - }
152 + // write true/false based on whether we got the same number or not.
153 + if (number == gw->prev_number) {
154 + if (hdr->nbits + 1 >= gw->capacity)
155 + return false;
156 +
157 + bit_buffer_write(data, hdr->nbits, static_cast<uint32_t>(1), 1);
158 + __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELEASE);
159 + __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELEASE);
160 + return true;
161 + }
162
150 - // write true/false based on whether we got the same number or not.
151 - if (number == gw->prev_number) {
163 if (hdr->nbits + 1 >= gw->capacity)
164 return false;
165 + bit_buffer_write(data, hdr->nbits, static_cast<uint32_t>(0), 1);
166 + __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELEASE);
167
155 - bit_buffer_write(data, hdr->nbits, static_cast<uint32_t>(1), 1);
156 - __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELAXED);
157 - __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELAXED);
158 - return true;
159 - }
168 + uint32_t xor_value = gw->prev_number ^ number;
169 + uint32_t xor_lzc = (bit_size<uint32_t>() == 32) ? __builtin_clz(xor_value) : __builtin_clzll(xor_value);
170 + uint32_t is_xor_lzc_same = (xor_lzc == gw->prev_xor_lzc) ? 1 : 0;
171
161 - if (hdr->nbits + 1 >= gw->capacity)
162 - return false;
163 - bit_buffer_write(data, hdr->nbits, static_cast<uint32_t>(0), 1);
164 - __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELAXED);
172 + if (hdr->nbits + 1 >= gw->capacity)
173 + return false;
174 + bit_buffer_write(data, hdr->nbits, is_xor_lzc_same, 1);
175 + __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELEASE);
176
166 - uint32_t xor_value = gw->prev_number ^ number;
167 - uint32_t xor_lzc = (bit_size<uint32_t>() == 32) ? __builtin_clz(xor_value) : __builtin_clzll(xor_value);
168 - uint32_t is_xor_lzc_same = (xor_lzc == gw->prev_xor_lzc) ? 1 : 0;
177 + if (!is_xor_lzc_same) {
178 + size_t bits_needed = (bit_size<uint32_t>() == 32) ? 5 : 6;
179 + if ((hdr->nbits + bits_needed) >= gw->capacity)
180 + return false;
181 + bit_buffer_write(data, hdr->nbits, xor_lzc, bits_needed);
182 + __atomic_fetch_add(&hdr->nbits, bits_needed, __ATOMIC_RELEASE);
183 + }
184
170 - if (hdr->nbits + 1 >= gw->capacity)
171 - return false;
172 - bit_buffer_write(data, hdr->nbits, is_xor_lzc_same, 1);
173 - __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELAXED);
174 -
175 - if (!is_xor_lzc_same) {
176 - size_t bits_needed = (bit_size<uint32_t>() == 32) ? 5 : 6;
177 - if ((hdr->nbits + bits_needed) >= gw->capacity)
185 + // write the bits of the XOR'd value without the LZC prefix
186 + if (hdr->nbits + (bit_size<uint32_t>() - xor_lzc) >= gw->capacity)
187 return false;
179 - bit_buffer_write(data, hdr->nbits, xor_lzc, bits_needed);
180 - __atomic_fetch_add(&hdr->nbits, bits_needed, __ATOMIC_RELAXED);
181 - }
182 -
183 - // write the bits of the XOR'd value without the LZC prefix
184 - if (hdr->nbits + (bit_size<uint32_t>() - xor_lzc) >= gw->capacity)
185 - return false;
186 - bit_buffer_write(data, hdr->nbits, xor_value, bit_size<uint32_t>() - xor_lzc);
187 - __atomic_fetch_add(&hdr->nbits, bit_size<uint32_t>() - xor_lzc, __ATOMIC_RELAXED);
188 - __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELAXED);
188 + bit_buffer_write(data, hdr->nbits, xor_value, bit_size<uint32_t>() - xor_lzc);
189 + __atomic_fetch_add(&hdr->nbits, bit_size<uint32_t>() - xor_lzc, __ATOMIC_RELEASE);
190 + __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELEASE);
191
190 - gw->prev_number = number;
191 - gw->prev_xor_lzc = xor_lzc;
192 - return true;
192 + gw->prev_number = number;
193 + gw->prev_xor_lzc = xor_lzc;
194 + return true;
195 + }
196 }
197
198 gorilla_buffer_t *gorilla_writer_drop_head_buffer(gorilla_writer_t *gw) {
@@ -198,7 +201,7 @@ gorilla_buffer_t *gorilla_writer_drop_head_buffer(gorilla_writer_t *gw) {
201
202 gorilla_buffer_t *curr_head = gw->head_buffer;
203 gorilla_buffer_t *next_head = gw->head_buffer->header.next;
201 - __atomic_store_n(&gw->head_buffer, next_head, __ATOMIC_RELAXED);
204 + __atomic_store_n(&gw->head_buffer, next_head, __ATOMIC_RELEASE);
205 return curr_head;
206 }
207
@@ -206,9 +209,9 @@ uint32_t gorilla_writer_actual_nbytes(const gorilla_writer_t *gw)
209 {
210 uint32_t nbytes = 0;
211
209 - const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_SEQ_CST);
212 + const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
213 do {
211 - const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_SEQ_CST);
214 + const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
215
216 nbytes += RRDENG_GORILLA_32BIT_BUFFER_SIZE;
217
@@ -222,14 +225,14 @@ uint32_t gorilla_writer_optimal_nbytes(const gorilla_writer_t *gw)
225 {
226 uint32_t nbytes = 0;
227
225 - const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_SEQ_CST);
228 + const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
229 do {
227 - const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_SEQ_CST);
230 + const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
231
232 if(next_gbuf)
233 nbytes += RRDENG_GORILLA_32BIT_BUFFER_SIZE;
234 else
232 - nbytes += gorilla_buffer_nbytes(__atomic_load_n(&curr_gbuf->header.nbits, __ATOMIC_SEQ_CST));
235 + nbytes += gorilla_buffer_nbytes(__atomic_load_n(&curr_gbuf->header.nbits, __ATOMIC_ACQUIRE));
236
237 curr_gbuf = next_gbuf;
238 } while (curr_gbuf);
@@ -312,10 +315,10 @@ size_t gorilla_buffer_unpatched_nbytes(const gorilla_buffer_t *gbuf) {
315
316 gorilla_reader_t gorilla_writer_get_reader(const gorilla_writer_t *gw)
317 {
315 - const gorilla_buffer_t *buffer = __atomic_load_n(&gw->head_buffer, __ATOMIC_SEQ_CST);
318 + const gorilla_buffer_t *buffer = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
319
317 - uint32_t entries = __atomic_load_n(&buffer->header.entries, __ATOMIC_SEQ_CST);
318 - uint32_t capacity = __atomic_load_n(&buffer->header.nbits, __ATOMIC_SEQ_CST);
320 + uint32_t entries = __atomic_load_n(&buffer->header.entries, __ATOMIC_ACQUIRE);
321 + uint32_t capacity = __atomic_load_n(&buffer->header.nbits, __ATOMIC_ACQUIRE);
322
323 return gorilla_reader_t {
324 .buffer = buffer,
@@ -331,8 +334,8 @@ gorilla_reader_t gorilla_writer_get_reader(const gorilla_writer_t *gw)
334
335 gorilla_reader_t gorilla_reader_init(gorilla_buffer_t *gbuf)
336 {
334 - uint32_t entries = __atomic_load_n(&gbuf->header.entries, __ATOMIC_SEQ_CST);
335 - uint32_t capacity = __atomic_load_n(&gbuf->header.nbits, __ATOMIC_SEQ_CST);
337 + uint32_t entries = __atomic_load_n(&gbuf->header.entries, __ATOMIC_ACQUIRE);
338 + uint32_t capacity = __atomic_load_n(&gbuf->header.nbits, __ATOMIC_ACQUIRE);
339
340 return gorilla_reader_t {
341 .buffer = gbuf,
@@ -346,79 +349,83 @@ gorilla_reader_t gorilla_reader_init(gorilla_buffer_t *gbuf)
349 };
350 }
351
349 -bool gorilla_reader_read(gorilla_reader_t *gr, uint32_t *number)
350 -{
351 - const uint32_t *data = gr->buffer->data;
352 -
353 - if (gr->index + 1 > gr->entries) {
354 - // We don't have any more entries to return. However, the writer
355 - // might have updated the buffer's entries. We need to check once
356 - // more in case more elements were added.
357 - gr->entries = __atomic_load_n(&gr->buffer->header.entries, __ATOMIC_SEQ_CST);
358 - gr->capacity = __atomic_load_n(&gr->buffer->header.nbits, __ATOMIC_SEQ_CST);
359 -
360 - // if the reader's current buffer has not been updated, we need to
361 - // check if it has a pointer to a next buffer.
362 - if (gr->index + 1 > gr->entries) {
363 - gorilla_buffer_t *next_buffer = __atomic_load_n(&gr->buffer->header.next, __ATOMIC_SEQ_CST);
364 -
365 - if (!next_buffer) {
366 - // fprintf(stderr, "Consumed reader with %zu entries from buffer %p\n (No more buffers to read from)", gr->length, gr->buffer);
367 - return false;
352 +extern "C" {
353 + ALWAYS_INLINE_ONLY bool gorilla_reader_read(gorilla_reader_t *gr, uint32_t *number)
354 + {
355 + const uint32_t *data = gr->buffer->data;
356 +
357 + while (gr->index + 1 > gr->entries) {
358 + // We don't have any more entries to return. However, the writer
359 + // might have updated the buffer's entries. We need to check once
360 + // more in case more elements were added.
361 + gr->entries = __atomic_load_n(&gr->buffer->header.entries, __ATOMIC_ACQUIRE);
362 + gr->capacity = __atomic_load_n(&gr->buffer->header.nbits, __ATOMIC_ACQUIRE);
363 +
364 + // if the reader's current buffer has not been updated, we need to
365 + // check if it has a pointer to a next buffer.
366 + if (gr->index + 1 > gr->entries) {
367 + gorilla_buffer_t *next_buffer = __atomic_load_n(&gr->buffer->header.next, __ATOMIC_ACQUIRE);
368 +
369 + if (!next_buffer) {
370 + // fprintf(stderr, "Consumed reader with %zu entries from buffer %p\n (No more buffers to read from)", gr->length, gr->buffer);
371 + return false;
372 + }
373 +
374 + // fprintf(stderr, "Consumed reader with %zu entries from buffer %p\n", gr->length, gr->buffer);
375 + *gr = gorilla_reader_init(next_buffer);
376 + data = gr->buffer->data;
377 }
369 -
370 - // fprintf(stderr, "Consumed reader with %zu entries from buffer %p\n", gr->length, gr->buffer);
371 - *gr = gorilla_reader_init(next_buffer);
372 - return gorilla_reader_read(gr, number);
378 + else
379 + break;
380 }
374 - }
381
376 - // read the first number
377 - if (gr->index == 0) {
378 - bit_buffer_read(data, gr->position, number, bit_size<uint32_t>());
382 + // read the first number
383 + if (gr->index == 0) {
384 + bit_buffer_read(data, gr->position, number, bit_size<uint32_t>());
385
380 - gr->index++;
381 - gr->position += bit_size<uint32_t>();
382 - gr->prev_number = *number;
383 - return true;
384 - }
386 + gr->index++;
387 + gr->position += bit_size<uint32_t>();
388 + gr->prev_number = *number;
389 + return true;
390 + }
391
386 - // process same-number bit
387 - uint32_t is_same_number;
388 - bit_buffer_read(data, gr->position, &is_same_number, 1);
389 - gr->position++;
392 + // process same-number bit
393 + uint32_t is_same_number;
394 + bit_buffer_read(data, gr->position, &is_same_number, 1);
395 + gr->position++;
396
391 - if (is_same_number) {
392 - *number = gr->prev_number;
393 - gr->index++;
394 - return true;
395 - }
397 + if (is_same_number) {
398 + *number = gr->prev_number;
399 + gr->index++;
400 + return true;
401 + }
402
397 - // proceess same-xor-lzc bit
398 - uint32_t xor_lzc = gr->prev_xor_lzc;
403 + // proceess same-xor-lzc bit
404 + uint32_t xor_lzc = gr->prev_xor_lzc;
405
400 - uint32_t same_xor_lzc;
401 - bit_buffer_read(data, gr->position, &same_xor_lzc, 1);
402 - gr->position++;
406 + uint32_t same_xor_lzc;
407 + bit_buffer_read(data, gr->position, &same_xor_lzc, 1);
408 + gr->position++;
409
404 - if (!same_xor_lzc) {
405 - bit_buffer_read(data, gr->position, &xor_lzc, (bit_size<uint32_t>() == 32) ? 5 : 6);
406 - gr->position += (bit_size<uint32_t>() == 32) ? 5 : 6;
407 - }
410 + if (!same_xor_lzc) {
411 + bit_buffer_read(data, gr->position, &xor_lzc, (bit_size<uint32_t>() == 32) ? 5 : 6);
412 + gr->position += (bit_size<uint32_t>() == 32) ? 5 : 6;
413 + }
414
409 - // process the non-lzc suffix
410 - uint32_t xor_value = 0;
411 - bit_buffer_read(data, gr->position, &xor_value, bit_size<uint32_t>() - xor_lzc);
412 - gr->position += bit_size<uint32_t>() - xor_lzc;
415 + // process the non-lzc suffix
416 + uint32_t xor_value = 0;
417 + bit_buffer_read(data, gr->position, &xor_value, bit_size<uint32_t>() - xor_lzc);
418 + gr->position += bit_size<uint32_t>() - xor_lzc;
419
414 - *number = (gr->prev_number ^ xor_value);
420 + *number = (gr->prev_number ^ xor_value);
421
416 - gr->index++;
417 - gr->prev_number = *number;
418 - gr->prev_xor_lzc = xor_lzc;
419 - gr->prev_xor = xor_value;
422 + gr->index++;
423 + gr->prev_number = *number;
424 + gr->prev_xor_lzc = xor_lzc;
425 + gr->prev_xor = xor_value;
426
421 - return true;
427 + return true;
428 + }
429 }
430
431 extern "C" {
@@ -428,9 +435,9 @@ void aral_unmark_allocation(struct aral *ar, void *ptr);
435
436 void gorilla_writer_aral_unmark(const gorilla_writer_t *gw, struct aral *ar)
437 {
431 - const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_SEQ_CST);
438 + const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
439 do {
433 - const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_SEQ_CST);
440 + const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
441
442 // Call the C function here
443 aral_unmark_allocation(ar, const_cast<void*>(static_cast<const void*>(curr_gbuf)));
src/libnetdata/log/nd_log.c
+1 -1
@@ -13,7 +13,7 @@ int aclklog_enabled = 0;
13
14 // --------------------------------------------------------------------------------------------------------------------
15
16 -void errno_clear(void) {
16 +ALWAYS_INLINE void errno_clear(void) {
17 errno = 0;
18
19 #if defined(OS_WINDOWS)
src/libnetdata/os/gettid.c
+1 -1
@@ -21,7 +21,7 @@ pid_t os_gettid(void) {
21 }
22
23 static __thread pid_t gettid_cached_tid = 0;
24 -pid_t gettid_cached(void) {
24 +ALWAYS_INLINE pid_t gettid_cached(void) {
25 if(unlikely(gettid_cached_tid == 0))
26 gettid_cached_tid = os_gettid();
27
src/libnetdata/storage_number/storage_number.c
+1 -1
@@ -74,7 +74,7 @@ bool is_system_ieee754_double(void) {
74 }
75 }
76
77 -storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {
77 +ALWAYS_INLINE storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {
78 // bit 32 = sign 0:positive, 1:negative
79 // bit 31 = 0:divide, 1:multiply
80 // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
src/libnetdata/storage_number/storage_number.h
+1 -1
@@ -130,7 +130,7 @@ static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) __attri
130 #define MAX_INCREMENTAL_PERCENT_RATE 10
131
132
133 -static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) {
133 +static ALWAYS_INLINE NETDATA_DOUBLE unpack_storage_number(storage_number value) {
134 extern NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];
135
136 if(unlikely(value == SN_EMPTY_SLOT))
src/web/api/queries/query.c
+2 -2
@@ -703,7 +703,7 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_TIME_GROUPING group_method)
703 }
704 }
705
706 -static inline void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_TIME_GROUPING add_flush) {
706 +static ALWAYS_INLINE void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_TIME_GROUPING add_flush) {
707 switch(add_flush) {
708 case RRDR_GROUPING_AVERAGE:
709 tg_average_add(r, value);
@@ -760,7 +760,7 @@ static inline void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_T
760 }
761 }
762
763 -static inline NETDATA_DOUBLE time_grouping_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr, const RRDR_TIME_GROUPING add_flush) {
763 +static ALWAYS_INLINE NETDATA_DOUBLE time_grouping_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr, const RRDR_TIME_GROUPING add_flush) {
764 switch(add_flush) {
765 case RRDR_GROUPING_AVERAGE:
766 return tg_average_flush(r, rrdr_value_options_ptr);