@cryptotaxi247 / netdata-1 / commits / 4aba1a3c0

better randomness for heartbeat (#18930)

* add 10ms buffering to streaming senders * queue for 1ms * added jitter to heartbeat and sender * remove jitter and add os_random in libnetdata * cleanup * cleanup * fix random for all operating systems * simpler random number generator * make microsleep wait at least 1 nsec * detect random number generators * use libnetdata os_random() * use the buffer versions of the random generators; fallback to using random() * provide random functions per size * getrandom() can fail; handle this case too * fix for wrong params

Costa Tsaousis committed Nov 4, 2024 at 13:55 UTC 4aba1a3c07797e80d58f3a1e4ddf5451c77e8b0d
13 files changed +326 -54
CMakeLists.txt
+18 -2
@@ -388,6 +388,9 @@ check_function_exists(sched_get_priority_max HAVE_SCHED_GET_PRIORITY_MAX)
388 check_function_exists(close_range HAVE_CLOSE_RANGE)
389 check_function_exists(backtrace HAVE_BACKTRACE)
390
391 +check_function_exists(arc4random_buf HAVE_ARC4RANDOM_BUF)
392 +check_function_exists(getrandom HAVE_GETRANDOM)
393 +
394 #
395 # check source compilation
396 #
@@ -539,6 +542,17 @@ int my_function() {
542 }
543 " HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT)
544
545 +# Windows MSVCRT random number generator
546 +# used only when compiling natively (not MSYS/CYGWIN)
547 +check_c_source_compiles("
548 + #define _CRT_RAND_S
549 + #include <stdlib.h>
550 + int main() {
551 + unsigned int x;
552 + return rand_s(&x);
553 + }
554 +" HAVE_RAND_S)
555 +
556 if(OS_FREEBSD OR OS_MACOS)
557 set(HAVE_BUILTIN_ATOMICS True)
558 endif()
@@ -759,8 +773,8 @@ set(LIBNETDATA_FILES
773 src/libnetdata/os/os-windows-wrappers.h
774 src/libnetdata/os/get_system_cpus.c
775 src/libnetdata/os/get_system_cpus.h
762 - src/libnetdata/os/tinysleep.c
763 - src/libnetdata/os/tinysleep.h
776 + src/libnetdata/os/sleep.c
777 + src/libnetdata/os/sleep.h
778 src/libnetdata/os/uuid_generate.c
779 src/libnetdata/os/uuid_generate.h
780 src/libnetdata/os/setenv.c
@@ -862,6 +876,8 @@ set(LIBNETDATA_FILES
876 src/libnetdata/log/nd_log-to-windows-common.h
877 src/libnetdata/common.h
878 src/libnetdata/xxHash/xxhash.h
879 + src/libnetdata/os/random.c
880 + src/libnetdata/os/random.h
881 )
882
883 if(ENABLE_PLUGIN_EBPF)
packaging/cmake/config.cmake.h.in
+4
@@ -69,6 +69,10 @@
69 #cmakedefine HAVE_DLSYM
70 #cmakedefine HAVE_LIBCURL
71
72 +#cmakedefine HAVE_ARC4RANDOM_BUF
73 +#cmakedefine HAVE_RAND_S
74 +#cmakedefine HAVE_GETRANDOM
75 +
76 #cmakedefine HAVE_BACKTRACE
77 #cmakedefine HAVE_CLOSE_RANGE
78 #cmakedefine HAVE_SCHED_GETSCHEDULER
src/aclk/aclk_util.c
+2 -4
@@ -342,15 +342,13 @@ unsigned long int aclk_tbeb_delay(int reset, int base, unsigned long int min, un
342
343 attempt++;
344
345 - if (attempt == 0) {
346 - srandom(time(NULL));
345 + if (attempt == 0)
346 return 0;
348 - }
347
348 unsigned long int delay = pow(base, attempt - 1);
349 delay *= MSEC_PER_SEC;
350
353 - delay += (random() % (MAX(1000, delay/2)));
351 + delay += (os_random32() % (MAX(1000, delay/2)));
352
353 if (delay <= min * MSEC_PER_SEC)
354 return min;
src/libnetdata/clocks/clocks.c
+13 -4
@@ -260,6 +260,9 @@ void sleep_to_absolute_time(usec_t usec) {
260 }
261 #endif
262
263 +#define HEARTBEAT_MIN_OFFSET_UT (150 * USEC_PER_MS)
264 +#define HEARTBEAT_RANDOM_OFFSET_UT (350 * USEC_PER_MS)
265 +
266 #define HEARTBEAT_ALIGNMENT_STATISTICS_SIZE 20
267 static SPINLOCK heartbeat_alignment_spinlock = NETDATA_SPINLOCK_INITIALIZER;
268 static size_t heartbeat_alignment_id = 0;
@@ -306,22 +309,27 @@ void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr,
309 memcpy(old, current, sizeof(struct heartbeat_thread_statistics) * HEARTBEAT_ALIGNMENT_STATISTICS_SIZE);
310 }
311
309 -static usec_t heartbeat_randomness(usec_t step __maybe_unused, size_t statistics_id) {
312 +static XXH64_hash_t heartbeat_hash(usec_t step, size_t statistics_id) {
313 struct {
314 + usec_t step;
315 pid_t pid;
316 pid_t tid;
317 usec_t now_ut;
318 size_t statistics_id;
319 char tag[ND_THREAD_TAG_MAX + 1];
320 } key = {
321 + .step = step,
322 .pid = getpid(),
323 .tid = os_gettid(),
324 .now_ut = now_realtime_usec(),
325 .statistics_id = statistics_id,
326 };
327 strncpyz(key.tag, nd_thread_tag(), sizeof(key.tag) - 1);
323 - XXH64_hash_t hash = XXH3_64bits(&key, sizeof(key));
324 - usec_t offset_ut = (100 * USEC_PER_MS) + (hash % (400 * USEC_PER_MS));
328 + return XXH3_64bits(&key, sizeof(key));
329 +}
330 +
331 +static usec_t heartbeat_randomness(XXH64_hash_t hash) {
332 + usec_t offset_ut = HEARTBEAT_MIN_OFFSET_UT + (hash % HEARTBEAT_RANDOM_OFFSET_UT);
333
334 // Calculate the scheduler tick interval in microseconds
335 usec_t scheduler_step_ut = USEC_PER_SEC / (usec_t)system_hz;
@@ -345,7 +353,8 @@ inline void heartbeat_init(heartbeat_t *hb, usec_t step) {
353
354 hb->step = step;
355 hb->realtime = 0ULL;
348 - hb->randomness = heartbeat_randomness(hb->step, hb->statistics_id);
356 + hb->hash = heartbeat_hash(hb->step, hb->statistics_id);
357 + hb->randomness = heartbeat_randomness(hb->hash);
358
359 if(hb->statistics_id < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE) {
360 heartbeat_alignment_values[hb->statistics_id].dt = 0;
src/libnetdata/clocks/clocks.h
+2
@@ -4,6 +4,7 @@
4 #define NETDATA_CLOCKS_H 1
5
6 #include "../libnetdata.h"
7 +#include "libnetdata/os/random.h"
8
9 #ifndef HAVE_CLOCK_GETTIME
10 struct timespec {
@@ -30,6 +31,7 @@ typedef struct heartbeat {
31 usec_t realtime;
32 usec_t randomness;
33 size_t statistics_id;
34 + XXH64_hash_t hash;
35 } heartbeat_t;
36
37 /* Linux value is as good as any other */
src/libnetdata/os/os.h
+2 -1
@@ -7,6 +7,7 @@
7 #include <sys/syscall.h>
8 #endif
9
10 +#include "random.h"
11 #include "timestamps.h"
12 #include "setproctitle.h"
13 #include "close_range.h"
@@ -17,7 +18,7 @@
18 #include "gettid.h"
19 #include "get_pid_max.h"
20 #include "get_system_cpus.h"
20 -#include "tinysleep.h"
21 +#include "sleep.h"
22 #include "uuid_generate.h"
23 #include "setenv.h"
24 #include "os-freebsd-wrappers.h"
src/libnetdata/os/random.c new
+214
@@ -0,0 +1,214 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "libnetdata/libnetdata.h"
4 +
5 +#if !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_RAND_S)
6 +static SPINLOCK random_lock = NETDATA_SPINLOCK_INITIALIZER;
7 +static __attribute__((constructor)) void seed_random() {
8 + // Use current time and process ID to create a high-entropy seed
9 + struct timeval tv;
10 + gettimeofday(&tv, NULL);
11 +
12 + uint32_t seed = (uint32_t)(tv.tv_sec ^ tv.tv_usec ^ getpid());
13 +
14 + // Seed the random number generator
15 + srandom(seed);
16 +}
17 +
18 +#if defined(HAVE_GETRANDOM)
19 +#include <sys/random.h>
20 +void getrandom_helper(void *buf, size_t buflen) {
21 + ssize_t result;
22 + while (buflen > 0) {
23 + result = getrandom(buf, buflen, 0);
24 + if (result == -1) {
25 + if (errno == EINTR) {
26 + // Interrupted, retry
27 + continue;
28 + } else if (errno == EAGAIN) {
29 + // Insufficient entropy; wait and retry
30 + tinysleep();
31 + continue;
32 + } else {
33 + // Fallback to using random() with a spinlock
34 + spinlock_lock(&random_lock);
35 + while (buflen > 0) {
36 + if (buflen >= sizeof(uint32_t)) {
37 + // Generate 4 bytes at a time
38 + uint32_t temp = random();
39 + memcpy(buf, &temp, sizeof(uint32_t));
40 + buf = (uint8_t *)buf + sizeof(uint32_t);
41 + buflen -= sizeof(uint32_t);
42 + } else if (buflen >= sizeof(uint16_t)) {
43 + // Generate 2 bytes at a time
44 + uint16_t temp = random();
45 + memcpy(buf, &temp, sizeof(uint16_t));
46 + buf = (uint8_t *)buf + sizeof(uint16_t);
47 + buflen -= sizeof(uint16_t);
48 + } else {
49 + // Generate remaining bytes
50 + uint32_t temp = random();
51 + for (size_t i = 0; i < buflen; i++) {
52 + ((uint8_t *)buf)[i] = temp & 0xFF;
53 + temp >>= 8;
54 + }
55 + buflen = 0;
56 + }
57 + }
58 + spinlock_unlock(&random_lock);
59 + return;
60 + }
61 + }
62 + buf = (uint8_t *)buf + result;
63 + buflen -= result;
64 + }
65 +}
66 +#endif // HAVE_GETRANDOM
67 +#endif // !HAVE_ARC4RANDOM_BUF && !HAVE_RAND_S
68 +
69 +// return a random number 0 to max - 1
70 +uint64_t os_random(uint64_t max) {
71 + if (max <= 1) return 0;
72 +
73 + uint64_t value;
74 +
75 +#if defined(HAVE_ARC4RANDOM_BUF)
76 + if (max <= UINT8_MAX) {
77 + uint8_t v;
78 + arc4random_buf(&v, sizeof(v));
79 + value = v;
80 + } else if(max <= UINT16_MAX) {
81 + uint16_t v;
82 + arc4random_buf(&v, sizeof(v));
83 + value = v;
84 + } else if (max <= UINT32_MAX) {
85 + uint32_t v;
86 + arc4random_buf(&v, sizeof(v));
87 + value = v;
88 + } else
89 + arc4random_buf(&value, sizeof(value));
90 +
91 +#elif defined(HAVE_RAND_S)
92 + if (max <= UINT_MAX) {
93 + unsigned int temp;
94 + rand_s(&temp);
95 + value = temp;
96 + } else {
97 + unsigned int temp_lo, temp_hi;
98 + rand_s(&temp_lo);
99 + rand_s(&temp_hi);
100 + value = ((uint64_t)temp_hi << 32) + (uint64_t)temp_lo;
101 + }
102 +
103 +#elif defined(HAVE_GETRANDOM)
104 + if (max <= UINT8_MAX) {
105 + uint8_t v;
106 + getrandom_helper(&v, sizeof(v));
107 + value = v;
108 + } else if(max <= UINT16_MAX) {
109 + uint16_t v;
110 + getrandom_helper(&v, sizeof(v));
111 + value = v;
112 + } else if (max <= UINT32_MAX) {
113 + uint32_t v;
114 + getrandom_helper(&v, sizeof(v));
115 + value = v;
116 + } else
117 + getrandom_helper(&value, sizeof(value));
118 +
119 +#else
120 + spinlock_lock(&random_lock);
121 + if(max <= INT32_MAX)
122 + value = random();
123 + else
124 + value = ((uint64_t) random() << 33) | ((uint64_t) random() << 2) | (random() & 0x3);
125 + spinlock_unlock(&random_lock);
126 +#endif
127 +
128 + return value % max;
129 +}
130 +
131 +// Generate an 8-bit random number
132 +uint8_t os_random8(void) {
133 + uint8_t value;
134 +
135 +#if defined(HAVE_ARC4RANDOM_BUF)
136 + arc4random_buf(&value, sizeof(value));
137 +#elif defined(HAVE_GETRANDOM)
138 + getrandom_helper(&value, sizeof(value));
139 +#elif defined(HAVE_RAND_S)
140 + unsigned int temp;
141 + rand_s(&temp);
142 + value = (uint8_t)temp;
143 +#else
144 + spinlock_lock(&random_lock);
145 + value = (uint8_t)random();
146 + spinlock_unlock(&random_lock);
147 +#endif
148 +
149 + return value;
150 +}
151 +
152 +// Generate a 16-bit random number
153 +uint16_t os_random16(void) {
154 + uint16_t value;
155 +
156 +#if defined(HAVE_ARC4RANDOM_BUF)
157 + arc4random_buf(&value, sizeof(value));
158 +#elif defined(HAVE_GETRANDOM)
159 + getrandom_helper(&value, sizeof(value));
160 +#elif defined(HAVE_RAND_S)
161 + unsigned int temp;
162 + rand_s(&temp);
163 + value = (uint16_t)temp;
164 +#else
165 + spinlock_lock(&random_lock);
166 + value = (uint16_t)random();
167 + spinlock_unlock(&random_lock);
168 +#endif
169 +
170 + return value;
171 +}
172 +
173 +// Generate a 32-bit random number
174 +uint32_t os_random32(void) {
175 + uint32_t value;
176 +
177 +#if defined(HAVE_ARC4RANDOM_BUF)
178 + arc4random_buf(&value, sizeof(value));
179 +#elif defined(HAVE_GETRANDOM)
180 + getrandom_helper(&value, sizeof(value));
181 +#elif defined(HAVE_RAND_S)
182 + unsigned int temp;
183 + rand_s(&temp);
184 + value = temp;
185 +#else
186 + spinlock_lock(&random_lock);
187 + value = random();
188 + spinlock_unlock(&random_lock);
189 +#endif
190 +
191 + return value;
192 +}
193 +
194 +// Generate a 64-bit random number
195 +uint64_t os_random64(void) {
196 + uint64_t value;
197 +
198 +#if defined(HAVE_ARC4RANDOM_BUF)
199 + arc4random_buf(&value, sizeof(value));
200 +#elif defined(HAVE_GETRANDOM)
201 + getrandom_helper(&value, sizeof(value));
202 +#elif defined(HAVE_RAND_S)
203 + unsigned int temp_lo, temp_hi;
204 + rand_s(&temp_lo);
205 + rand_s(&temp_hi);
206 + value = ((uint64_t)temp_hi << 32) | (uint64_t)temp_lo;
207 +#else
208 + spinlock_lock(&random_lock);
209 + value = ((uint64_t)random() << 33) | ((uint64_t)random() << 2) | (random() & 0x3);
210 + spinlock_unlock(&random_lock);
211 +#endif
212 +
213 + return value;
214 +}
src/libnetdata/os/random.h new
+16
@@ -0,0 +1,16 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RANDOM_H
4 +#define NETDATA_RANDOM_H
5 +
6 +#include "libnetdata/common.h"
7 +
8 +// return a random number 0 to max - 1
9 +uint64_t os_random(uint64_t max);
10 +
11 +uint8_t os_random8(void);
12 +uint16_t os_random16(void);
13 +uint32_t os_random32(void);
14 +uint64_t os_random64(void);
15 +
16 +#endif //NETDATA_RANDOM_H
src/libnetdata/os/sleep.c new
+36
@@ -0,0 +1,36 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "../libnetdata.h"
4 +
5 +#ifdef OS_WINDOWS
6 +void tinysleep(void) {
7 + Sleep(1);
8 +}
9 +#else
10 +void tinysleep(void) {
11 + static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
12 + nanosleep(&ns, NULL);
13 +}
14 +#endif
15 +
16 +#ifdef OS_WINDOWS
17 +void microsleep(usec_t ut) {
18 + size_t ms = ut / USEC_PER_MS + ((ut == 0 || (ut % USEC_PER_MS)) ? 1 : 0);
19 + Sleep(ms);
20 +}
21 +#else
22 +void microsleep(usec_t ut) {
23 + time_t secs = (time_t)(ut / USEC_PER_SEC);
24 + nsec_t nsec = (ut % USEC_PER_SEC) * NSEC_PER_USEC + ((ut == 0) ? 1 : 0);
25 +
26 + struct timespec remaining = {
27 + .tv_sec = secs,
28 + .tv_nsec = nsec,
29 + };
30 +
31 + errno_clear();
32 + while (nanosleep(&remaining, &remaining) == -1 && errno == EINTR && (remaining.tv_sec || remaining.tv_nsec)) {
33 + // Loop continues if interrupted by a signal
34 + }
35 +}
36 +#endif
src/libnetdata/os/sleep.h new
+9
@@ -0,0 +1,9 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_SLEEP_H
4 +#define NETDATA_SLEEP_H
5 +
6 +void tinysleep(void);
7 +void microsleep(usec_t ut);
8 +
9 +#endif //NETDATA_SLEEP_H
src/libnetdata/os/tinysleep.c deleted
-21
@@ -1,21 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "../libnetdata.h"
4 -
5 -#ifdef OS_WINDOWS
6 -void tinysleep(void) {
7 - // Improve the system timer resolution to 1 ms
8 - timeBeginPeriod(1);
9 -
10 - // Sleep for the desired duration
11 - Sleep(1);
12 -
13 - // Reset the system timer resolution
14 - timeEndPeriod(1);
15 -}
16 -#else
17 -void tinysleep(void) {
18 - static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
19 - nanosleep(&ns, NULL);
20 -}
21 -#endif
src/libnetdata/os/tinysleep.h deleted
-8
@@ -1,8 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_TINYSLEEP_H
4 -#define NETDATA_TINYSLEEP_H
5 -
6 -void tinysleep(void);
7 -
8 -#endif //NETDATA_TINYSLEEP_H
src/streaming/compression.c
+10 -14
@@ -395,21 +395,17 @@ size_t rrdpush_decompress(struct decompressor_state *state, const char *compress
395 // ----------------------------------------------------------------------------
396 // unit test
397
398 -static inline long int my_random (void) {
399 - return random();
400 -}
401 -
398 void unittest_generate_random_name(char *dst, size_t size) {
399 if(size < 7)
400 size = 7;
401
406 - size_t len = 5 + my_random() % (size - 6);
402 + size_t len = 5 + os_random32() % (size - 6);
403
404 for(size_t i = 0; i < len ; i++) {
409 - if(my_random() % 2 == 0)
410 - dst[i] = 'A' + my_random() % 26;
405 + if(os_random8() % 2 == 0)
406 + dst[i] = 'A' + os_random8() % 26;
407 else
412 - dst[i] = 'a' + my_random() % 26;
408 + dst[i] = 'a' + os_random8() % 26;
409 }
410
411 dst[len] = '\0';
@@ -423,9 +419,9 @@ void unittest_generate_message(BUFFER *wb, time_t now_s, size_t counter) {
419 time_t point_end_time_s = now_s;
420 time_t wall_clock_time_s = now_s;
421 size_t chart_slot = counter + 1;
426 - size_t dimensions = 2 + my_random() % 5;
422 + size_t dimensions = 2 + os_random8() % 5;
423 char chart[RRD_ID_LENGTH_MAX + 1] = "name";
428 - unittest_generate_random_name(chart, 5 + my_random() % 30);
424 + unittest_generate_random_name(chart, 5 + os_random8() % 30);
425
426 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2, sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1);
427
@@ -451,10 +447,10 @@ void unittest_generate_message(BUFFER *wb, time_t now_s, size_t counter) {
447 for(size_t d = 0; d < dimensions ;d++) {
448 size_t dim_slot = d + 1;
449 char dim_id[RRD_ID_LENGTH_MAX + 1] = "dimension";
454 - unittest_generate_random_name(dim_id, 10 + my_random() % 20);
455 - int64_t last_collected_value = (my_random() % 2 == 0) ? (int64_t)(counter + d) : (int64_t)my_random();
456 - NETDATA_DOUBLE value = (my_random() % 2 == 0) ? (NETDATA_DOUBLE)my_random() / ((NETDATA_DOUBLE)my_random() + 1) : (NETDATA_DOUBLE)last_collected_value;
457 - SN_FLAGS flags = (my_random() % 1000 == 0) ? SN_FLAG_NONE : SN_FLAG_NOT_ANOMALOUS;
450 + unittest_generate_random_name(dim_id, 10 + os_random8() % 20);
451 + int64_t last_collected_value = (os_random8() % 2 == 0) ? (int64_t)(counter + d) : (int64_t)os_random32();
452 + NETDATA_DOUBLE value = (os_random8() % 2 == 0) ? (NETDATA_DOUBLE)os_random64() / ((NETDATA_DOUBLE)os_random64() + 1) : (NETDATA_DOUBLE)last_collected_value;
453 + SN_FLAGS flags = (os_random16() % 1000 == 0) ? SN_FLAG_NONE : SN_FLAG_NOT_ANOMALOUS;
454
455 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2, sizeof(PLUGINSD_KEYWORD_SET_V2) - 1);
456