@cryptotaxi247 / netdata-1 / commits / 35eeffd8e

Add library to encode/decode Gorilla compressed buffers. (#15128)

* Add library to encode/decode Gorilla compressed buffers. * Code cleanup + fix high-level API for 64 bits. * Add scripts to build benchmarks and fuzzer. * Fix CMake builds * Add license note. * Return 0 instead of false literal.

vkalintiris committed Jun 8, 2023 at 07:41 UTC 35eeffd8e123465aca00c3464b8e1031b3f450e6
8 files changed +718
.gitignore
+5
@@ -232,3 +232,8 @@ Session.*.vim
232 # m4 generated ksys
233 database/engine/journalfile_v2.ksy
234 database/engine/journalfile_v2_virtmemb.ksy
235 +
236 +# gorilla benchmark & fuzz binaries
237 +libnetdata/gorilla/gorilla_benchmark
238 +libnetdata/gorilla/gorilla_fuzzer
239 +libnetdata/gorilla/fuzz-*.log
CMakeLists.txt
+2
@@ -446,6 +446,8 @@ set(LIBNETDATA_FILES
446 libnetdata/dictionary/dictionary.h
447 libnetdata/eval/eval.c
448 libnetdata/eval/eval.h
449 + libnetdata/gorilla/gorilla.cc
450 + libnetdata/gorilla/gorilla.h
451 libnetdata/health/health.c
452 libnetdata/health/health.h
453 libnetdata/july/july.c
Makefile.am
+2
@@ -154,6 +154,8 @@ LIBNETDATA_FILES = \
154 libnetdata/dictionary/dictionary.h \
155 libnetdata/eval/eval.c \
156 libnetdata/eval/eval.h \
157 + libnetdata/gorilla/gorilla.h \
158 + libnetdata/gorilla/gorilla.cc \
159 libnetdata/inlined.h \
160 libnetdata/july/july.c \
161 libnetdata/july/july.h \
libnetdata/gorilla/benchmark.sh new
+14
@@ -0,0 +1,14 @@
1 +#!/usr/bin/env bash
2 +#
3 +# SPDX-License-Identifier: GPL-3.0-or-later
4 +#
5 +
6 +set -exu -o pipefail
7 +
8 +clang++ \
9 + -std=c++11 -Wall -Wextra \
10 + -DENABLE_BENCHMARK -O2 -g \
11 + -lbenchmark -lbenchmark_main \
12 + -o gorilla_benchmark gorilla.cc
13 +
14 +./gorilla_benchmark
libnetdata/gorilla/fuzzer.sh new
+14
@@ -0,0 +1,14 @@
1 +#!/usr/bin/env bash
2 +#
3 +# SPDX-License-Identifier: GPL-3.0-or-later
4 +#
5 +
6 +set -exu -o pipefail
7 +
8 +clang++ \
9 + -std=c++11 -Wall -Wextra \
10 + -DENABLE_FUZZER -O2 -g \
11 + -fsanitize=fuzzer \
12 + -o gorilla_fuzzer gorilla.cc
13 +
14 +./gorilla_fuzzer -workers=8 -jobs=8
libnetdata/gorilla/gorilla.cc new
+620
@@ -0,0 +1,620 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "gorilla.h"
4 +
5 +#include <cassert>
6 +#include <climits>
7 +#include <cstdio>
8 +#include <cstring>
9 +
10 +using std::size_t;
11 +
12 +template <typename T>
13 +static constexpr size_t bit_size() noexcept
14 +{
15 + static_assert((sizeof(T) * CHAR_BIT) == 32 || (sizeof(T) * CHAR_BIT) == 64,
16 + "Word size should be 32 or 64 bits.");
17 + return (sizeof(T) * CHAR_BIT);
18 +}
19 +
20 +/*
21 + * Low-level bitstream operations, allowing us to read/write individual bits.
22 +*/
23 +
24 +template<typename Word>
25 +struct bit_stream_t {
26 + Word *buffer;
27 + size_t capacity;
28 + size_t position;
29 +};
30 +
31 +template<typename Word>
32 +static bit_stream_t<Word> bit_stream_new(Word *buffer, Word capacity) {
33 + bit_stream_t<Word> bs;
34 +
35 + bs.buffer = buffer;
36 + bs.capacity = capacity * bit_size<Word>();
37 + bs.position = 0;
38 +
39 + return bs;
40 +}
41 +
42 +template<typename Word>
43 +static bool bit_stream_write(bit_stream_t<Word> *bs, Word value, size_t nbits) {
44 + assert(nbits > 0 && nbits <= bit_size<Word>());
45 + assert(bs->capacity >= (bs->position + nbits));
46 +
47 + if (bs->position + nbits > bs->capacity) {
48 + return false;
49 + }
50 +
51 + const size_t index = bs->position / bit_size<Word>();
52 + const size_t offset = bs->position % bit_size<Word>();
53 + bs->position += nbits;
54 +
55 + if (offset == 0) {
56 + bs->buffer[index] = value;
57 + } else {
58 + const size_t remaining_bits = bit_size<Word>() - offset;
59 +
60 + // write the lower part of the value
61 + const Word low_bits_mask = ((Word) 1 << remaining_bits) - 1;
62 + const Word lowest_bits_in_value = value & low_bits_mask;
63 + bs->buffer[index] |= (lowest_bits_in_value << offset);
64 +
65 + if (nbits > remaining_bits) {
66 + // write the upper part of the value
67 + const Word high_bits_mask = ~low_bits_mask;
68 + const Word highest_bits_in_value = (value & high_bits_mask) >> (remaining_bits);
69 + bs->buffer[index + 1] = highest_bits_in_value;
70 + }
71 + }
72 +
73 + return true;
74 +}
75 +
76 +template<typename Word>
77 +static bool bit_stream_read(bit_stream_t<Word> *bs, Word *value, size_t nbits) {
78 + assert(nbits > 0 && nbits <= bit_size<Word>());
79 + assert(bs->capacity >= (bs->position + nbits));
80 +
81 + if (bs->position + nbits > bs->capacity) {
82 + return false;
83 + }
84 +
85 + const size_t index = bs->position / bit_size<Word>();
86 + const size_t offset = bs->position % bit_size<Word>();
87 + bs->position += nbits;
88 +
89 + if (offset == 0) {
90 + *value = (nbits == bit_size<Word>()) ?
91 + bs->buffer[index] :
92 + bs->buffer[index] & (((Word) 1 << nbits) - 1);
93 + } else {
94 + const size_t remaining_bits = bit_size<Word>() - offset;
95 +
96 + // extract the lower part of the value
97 + if (nbits < remaining_bits) {
98 + *value = (bs->buffer[index] >> offset) & (((Word) 1 << nbits) - 1);
99 + } else {
100 + *value = (bs->buffer[index] >> offset) & (((Word) 1 << remaining_bits) - 1);
101 + nbits -= remaining_bits;
102 + *value |= (bs->buffer[index + 1] & (((Word) 1 << nbits) - 1)) << remaining_bits;
103 + }
104 + }
105 +
106 + return true;
107 +}
108 +
109 +/*
110 + * High-level Gorilla codec implementation
111 +*/
112 +
113 +template<typename Word>
114 +struct bit_code_t {
115 + bit_stream_t<Word> bs;
116 + Word entries;
117 + Word prev_number;
118 + Word prev_xor;
119 + Word prev_xor_lzc;
120 +};
121 +
122 +template<typename Word>
123 +static void bit_code_init(bit_code_t<Word> *bc, Word *buffer, Word capacity) {
124 + bc->bs = bit_stream_new(buffer, capacity);
125 +
126 + bc->entries = 0;
127 + bc->prev_number = 0;
128 + bc->prev_xor = 0;
129 + bc->prev_xor_lzc = 0;
130 +
131 + // reserved two words:
132 + // Buffer[0] -> number of entries written
133 + // Buffer[1] -> number of bits written
134 +
135 + bc->bs.position += 2 * bit_size<Word>();
136 +}
137 +
138 +template<typename Word>
139 +static bool bit_code_read(bit_code_t<Word> *bc, Word *number) {
140 + bit_stream_t<Word> *bs = &bc->bs;
141 +
142 + bc->entries++;
143 +
144 + // read the first number
145 + if (bc->entries == 1) {
146 + bool ok = bit_stream_read(bs, number, bit_size<Word>());
147 + bc->prev_number = *number;
148 + return ok;
149 + }
150 +
151 + // process same-number bit
152 + Word is_same_number;
153 + if (!bit_stream_read(bs, &is_same_number, 1)) {
154 + return false;
155 + }
156 +
157 + if (is_same_number) {
158 + *number = bc->prev_number;
159 + return true;
160 + }
161 +
162 + // proceess same-xor-lzc bit
163 + Word xor_lzc = bc->prev_xor_lzc;
164 +
165 + Word same_xor_lzc;
166 + if (!bit_stream_read(bs, &same_xor_lzc, 1)) {
167 + return false;
168 + }
169 +
170 + if (!same_xor_lzc) {
171 + if (!bit_stream_read(bs, &xor_lzc, (bit_size<Word>() == 32) ? 5 : 6)) {
172 + return false;
173 + }
174 + }
175 +
176 + // process the non-lzc suffix
177 + Word xor_value = 0;
178 + if (!bit_stream_read(bs, &xor_value, bit_size<Word>() - xor_lzc)) {
179 + return false;
180 + }
181 +
182 + *number = (bc->prev_number ^ xor_value);
183 +
184 + bc->prev_number = *number;
185 + bc->prev_xor_lzc = xor_lzc;
186 + bc->prev_xor = xor_value;
187 +
188 + return true;
189 +}
190 +
191 +template<typename Word>
192 +static bool bit_code_write(bit_code_t<Word> *bc, const Word number) {
193 + bit_stream_t<Word> *bs = &bc->bs;
194 + Word position = bs->position;
195 +
196 + bc->entries++;
197 +
198 + // this is the first number we are writing
199 + if (bc->entries == 1) {
200 + bc->prev_number = number;
201 + return bit_stream_write(bs, number, bit_size<Word>());
202 + }
203 +
204 + // write true/false based on whether we got the same number or not.
205 + if (number == bc->prev_number) {
206 + return bit_stream_write(bs, static_cast<Word>(1), 1);
207 + } else {
208 + if (bit_stream_write(bs, static_cast<Word>(0), 1) == false) {
209 + return false;
210 + }
211 + }
212 +
213 + // otherwise:
214 + // - compute the non-zero xor
215 + // - find its leading-zero count
216 +
217 + Word xor_value = bc->prev_number ^ number;
218 + // FIXME: Use SFINAE
219 + Word xor_lzc = (bit_size<Word>() == 32) ? __builtin_clz(xor_value) : __builtin_clzll(xor_value);
220 + Word is_xor_lzc_same = (xor_lzc == bc->prev_xor_lzc) ? 1 : 0;
221 +
222 + if (is_xor_lzc_same) {
223 + // xor-lzc is same
224 + if (bit_stream_write(bs, static_cast<Word>(1), 1) == false) {
225 + goto RET_FALSE;
226 + }
227 + } else {
228 + // xor-lzc is different
229 + if (bit_stream_write(bs, static_cast<Word>(0), 1) == false) {
230 + goto RET_FALSE;
231 + }
232 +
233 + if (bit_stream_write(bs, xor_lzc, (bit_size<Word>() == 32) ? 5 : 6) == false) {
234 + goto RET_FALSE;
235 + }
236 + }
237 +
238 + // write the bits of the XOR value without the LZC prefix
239 + if (bit_stream_write(bs, xor_value, bit_size<Word>() - xor_lzc) == false) {
240 + goto RET_FALSE;
241 + }
242 +
243 + bc->prev_number = number;
244 + bc->prev_xor_lzc = xor_lzc;
245 + return true;
246 +
247 +RET_FALSE:
248 + bc->bs.position = position;
249 + return false;
250 +}
251 +
252 +// only valid for writers
253 +template<typename Word>
254 +static bool bit_code_flush(bit_code_t<Word> *bc) {
255 + bit_stream_t<Word> *bs = &bc->bs;
256 +
257 + Word num_entries_written = bc->entries;
258 + Word num_bits_written = bs->position;
259 +
260 + // we want to write these at the beginning
261 + bs->position = 0;
262 +
263 + if (!bit_stream_write(bs, num_entries_written, bit_size<Word>())) {
264 + return false;
265 + }
266 +
267 + if (!bit_stream_write(bs, num_bits_written, bit_size<Word>())) {
268 + return false;
269 + }
270 +
271 + bs->position = num_bits_written;
272 + return true;
273 +}
274 +
275 +// only valid for readers
276 +template<typename Word>
277 +static bool bit_code_info(bit_code_t<Word> *bc, Word *num_entries_written,
278 + Word *num_bits_written) {
279 + bit_stream_t<Word> *bs = &bc->bs;
280 +
281 + assert(bs->position == 2 * bit_size<Word>());
282 + if (bs->capacity < (2 * bit_size<Word>())) {
283 + return false;
284 + }
285 +
286 + if (num_entries_written) {
287 + *num_entries_written = bs->buffer[0];
288 + }
289 + if (num_bits_written) {
290 + *num_bits_written = bs->buffer[1];
291 + }
292 +
293 + return true;
294 +}
295 +
296 +template<typename Word>
297 +static size_t gorilla_encode(Word *dst, Word dst_len, const Word *src, Word src_len) {
298 + bit_code_t<Word> bcw;
299 +
300 + bit_code_init(&bcw, dst, dst_len);
301 +
302 + for (size_t i = 0; i != src_len; i++) {
303 + if (!bit_code_write(&bcw, src[i]))
304 + return 0;
305 + }
306 +
307 + if (!bit_code_flush(&bcw))
308 + return 0;
309 +
310 + return src_len;
311 +}
312 +
313 +template<typename Word>
314 +static size_t gorilla_decode(Word *dst, Word dst_len, const Word *src, Word src_len) {
315 + bit_code_t<Word> bcr;
316 +
317 + bit_code_init(&bcr, (Word *) src, src_len);
318 +
319 + Word num_entries;
320 + if (!bit_code_info(&bcr, &num_entries, (Word *) NULL)) {
321 + return 0;
322 + }
323 + if (num_entries > dst_len) {
324 + return 0;
325 + }
326 +
327 + for (size_t i = 0; i != num_entries; i++) {
328 + if (!bit_code_read(&bcr, &dst[i]))
329 + return 0;
330 + }
331 +
332 + return num_entries;
333 +}
334 +
335 +/*
336 + * Low-level public API
337 +*/
338 +
339 +// 32-bit API
340 +
341 +void bit_code_writer_u32_init(bit_code_writer_u32_t *bcw, uint32_t *buffer, uint32_t capacity) {
342 + bit_code_t<uint32_t> *bc = (bit_code_t<uint32_t> *) bcw;
343 + bit_code_init(bc, buffer, capacity);
344 +}
345 +
346 +bool bit_code_writer_u32_write(bit_code_writer_u32_t *bcw, const uint32_t number) {
347 + bit_code_t<uint32_t> *bc = (bit_code_t<uint32_t> *) bcw;
348 + return bit_code_write(bc, number);
349 +}
350 +
351 +bool bit_code_writer_u32_flush(bit_code_writer_u32_t *bcw) {
352 + bit_code_t<uint32_t> *bc = (bit_code_t<uint32_t> *) bcw;
353 + return bit_code_flush(bc);
354 +}
355 +
356 +void bit_code_reader_u32_init(bit_code_reader_u32_t *bcr, uint32_t *buffer, uint32_t capacity) {
357 + bit_code_t<uint32_t> *bc = (bit_code_t<uint32_t> *) bcr;
358 + bit_code_init(bc, buffer, capacity);
359 +}
360 +
361 +bool bit_code_reader_u32_read(bit_code_reader_u32_t *bcr, uint32_t *number) {
362 + bit_code_t<uint32_t> *bc = (bit_code_t<uint32_t> *) bcr;
363 + return bit_code_read(bc, number);
364 +}
365 +
366 +bool bit_code_reader_u32_info(bit_code_reader_u32_t *bcr, uint32_t *num_entries_written,
367 + uint32_t *num_bits_written) {
368 + bit_code_t<uint32_t> *bc = (bit_code_t<uint32_t> *) bcr;
369 + return bit_code_info(bc, num_entries_written, num_bits_written);
370 +}
371 +
372 +// 64-bit API
373 +
374 +void bit_code_writer_u64_init(bit_code_writer_u64_t *bcw, uint64_t *buffer, uint64_t capacity) {
375 + bit_code_t<uint64_t> *bc = (bit_code_t<uint64_t> *) bcw;
376 + bit_code_init(bc, buffer, capacity);
377 +}
378 +
379 +bool bit_code_writer_u64_write(bit_code_writer_u64_t *bcw, const uint64_t number) {
380 + bit_code_t<uint64_t> *bc = (bit_code_t<uint64_t> *) bcw;
381 + return bit_code_write(bc, number);
382 +}
383 +
384 +bool bit_code_writer_u64_flush(bit_code_writer_u64_t *bcw) {
385 + bit_code_t<uint64_t> *bc = (bit_code_t<uint64_t> *) bcw;
386 + return bit_code_flush(bc);
387 +}
388 +
389 +void bit_code_reader_u64_init(bit_code_reader_u64_t *bcr, uint64_t *buffer, uint64_t capacity) {
390 + bit_code_t<uint64_t> *bc = (bit_code_t<uint64_t> *) bcr;
391 + bit_code_init(bc, buffer, capacity);
392 +}
393 +
394 +bool bit_code_reader_u64_read(bit_code_reader_u64_t *bcr, uint64_t *number) {
395 + bit_code_t<uint64_t> *bc = (bit_code_t<uint64_t> *) bcr;
396 + return bit_code_read(bc, number);
397 +}
398 +
399 +bool bit_code_reader_u64_info(bit_code_reader_u64_t *bcr, uint64_t *num_entries_written,
400 + uint64_t *num_bits_written) {
401 + bit_code_t<uint64_t> *bc = (bit_code_t<uint64_t> *) bcr;
402 + return bit_code_info(bc, num_entries_written, num_bits_written);
403 +}
404 +
405 +/*
406 + * High-level public API
407 +*/
408 +
409 +// 32-bit API
410 +
411 +size_t gorilla_encode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len) {
412 + return gorilla_encode(dst, (uint32_t) dst_len, src, (uint32_t) src_len);
413 +}
414 +
415 +size_t gorilla_decode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len) {
416 + return gorilla_decode(dst, (uint32_t) dst_len, src, (uint32_t) src_len);
417 +}
418 +
419 +// 64-bit API
420 +
421 +size_t gorilla_encode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len) {
422 + return gorilla_encode(dst, (uint64_t) dst_len, src, (uint64_t) src_len);
423 +}
424 +
425 +size_t gorilla_decode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len) {
426 + return gorilla_decode(dst, (uint64_t) dst_len, src, (uint64_t) src_len);
427 +}
428 +
429 +/*
430 + * Internal code used for fuzzing the library
431 +*/
432 +
433 +#ifdef ENABLE_FUZZER
434 +
435 +#include <vector>
436 +
437 +template<typename Word>
438 +static std::vector<Word> random_vector(const uint8_t *data, size_t size) {
439 + std::vector<Word> V;
440 +
441 + V.reserve(1024);
442 +
443 + while (size >= sizeof(Word)) {
444 + size -= sizeof(Word);
445 +
446 + Word w;
447 + memcpy(&w, &data[size], sizeof(Word));
448 + V.push_back(w);
449 + }
450 +
451 + return V;
452 +}
453 +
454 +template<typename Word>
455 +static void check_equal_buffers(Word *lhs, Word lhs_size, Word *rhs, Word rhs_size) {
456 + assert((lhs_size == rhs_size) && "Buffers have different size.");
457 +
458 + for (size_t i = 0; i != lhs_size; i++) {
459 + assert((lhs[i] == rhs[i]) && "Buffers differ");
460 + }
461 +}
462 +
463 +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
464 + // 32-bit tests
465 + {
466 + if (Size < 4)
467 + return 0;
468 +
469 + std::vector<uint32_t> RandomData = random_vector<uint32_t>(Data, Size);
470 + std::vector<uint32_t> EncodedData(10 * RandomData.capacity(), 0);
471 + std::vector<uint32_t> DecodedData(10 * RandomData.capacity(), 0);
472 +
473 + size_t num_entries_written = gorilla_encode_u32(EncodedData.data(), EncodedData.size(),
474 + RandomData.data(), RandomData.size());
475 + size_t num_entries_read = gorilla_decode_u32(DecodedData.data(), DecodedData.size(),
476 + EncodedData.data(), EncodedData.size());
477 +
478 + assert(num_entries_written == num_entries_read);
479 + check_equal_buffers(RandomData.data(), (uint32_t) RandomData.size(),
480 + DecodedData.data(), (uint32_t) RandomData.size());
481 + }
482 +
483 + // 64-bit tests
484 + {
485 + if (Size < 8)
486 + return 0;
487 +
488 + std::vector<uint64_t> RandomData = random_vector<uint64_t>(Data, Size);
489 + std::vector<uint64_t> EncodedData(10 * RandomData.capacity(), 0);
490 + std::vector<uint64_t> DecodedData(10 * RandomData.capacity(), 0);
491 +
492 + size_t num_entries_written = gorilla_encode_u64(EncodedData.data(), EncodedData.size(),
493 + RandomData.data(), RandomData.size());
494 + size_t num_entries_read = gorilla_decode_u64(DecodedData.data(), DecodedData.size(),
495 + EncodedData.data(), EncodedData.size());
496 +
497 + assert(num_entries_written == num_entries_read);
498 + check_equal_buffers(RandomData.data(), (uint64_t) RandomData.size(),
499 + DecodedData.data(), (uint64_t) RandomData.size());
500 + }
501 +
502 + return 0;
503 +}
504 +
505 +#endif /* ENABLE_FUZZER */
506 +
507 +#ifdef ENABLE_BENCHMARK
508 +
509 +#include <benchmark/benchmark.h>
510 +#include <random>
511 +
512 +static size_t NumItems = 1024;
513 +
514 +static void BM_EncodeU32Numbers(benchmark::State& state) {
515 + std::random_device rd;
516 + std::mt19937 mt(rd());
517 + std::uniform_int_distribution<uint32_t> dist(0x0, 0x0000FFFF);
518 +
519 + std::vector<uint32_t> RandomData;
520 + for (size_t idx = 0; idx != NumItems; idx++) {
521 + RandomData.push_back(dist(mt));
522 + }
523 + std::vector<uint32_t> EncodedData(10 * RandomData.capacity(), 0);
524 +
525 + for (auto _ : state) {
526 + benchmark::DoNotOptimize(
527 + gorilla_encode_u32(EncodedData.data(), EncodedData.size(),
528 + RandomData.data(), RandomData.size())
529 + );
530 + benchmark::ClobberMemory();
531 + }
532 +
533 + state.SetItemsProcessed(NumItems * state.iterations());
534 + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint32_t));
535 +}
536 +BENCHMARK(BM_EncodeU32Numbers);
537 +
538 +static void BM_DecodeU32Numbers(benchmark::State& state) {
539 + std::random_device rd;
540 + std::mt19937 mt(rd());
541 + std::uniform_int_distribution<uint32_t> dist(0x0, 0xFFFFFFFF);
542 +
543 + std::vector<uint32_t> RandomData;
544 + for (size_t idx = 0; idx != NumItems; idx++) {
545 + RandomData.push_back(dist(mt));
546 + }
547 + std::vector<uint32_t> EncodedData(10 * RandomData.capacity(), 0);
548 + std::vector<uint32_t> DecodedData(10 * RandomData.capacity(), 0);
549 +
550 + gorilla_encode_u32(EncodedData.data(), EncodedData.size(),
551 + RandomData.data(), RandomData.size());
552 +
553 + for (auto _ : state) {
554 + benchmark::DoNotOptimize(
555 + gorilla_decode_u32(DecodedData.data(), DecodedData.size(),
556 + EncodedData.data(), EncodedData.size())
557 + );
558 + benchmark::ClobberMemory();
559 + }
560 +
561 + state.SetItemsProcessed(NumItems * state.iterations());
562 + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint32_t));
563 +}
564 +// Register the function as a benchmark
565 +BENCHMARK(BM_DecodeU32Numbers);
566 +
567 +static void BM_EncodeU64Numbers(benchmark::State& state) {
568 + std::random_device rd;
569 + std::mt19937 mt(rd());
570 + std::uniform_int_distribution<uint64_t> dist(0x0, 0x0000FFFF);
571 +
572 + std::vector<uint64_t> RandomData;
573 + for (size_t idx = 0; idx != 1024; idx++) {
574 + RandomData.push_back(dist(mt));
575 + }
576 + std::vector<uint64_t> EncodedData(10 * RandomData.capacity(), 0);
577 +
578 + for (auto _ : state) {
579 + benchmark::DoNotOptimize(
580 + gorilla_encode_u64(EncodedData.data(), EncodedData.size(),
581 + RandomData.data(), RandomData.size())
582 + );
583 + benchmark::ClobberMemory();
584 + }
585 +
586 + state.SetItemsProcessed(NumItems * state.iterations());
587 + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint64_t));
588 +}
589 +BENCHMARK(BM_EncodeU64Numbers);
590 +
591 +static void BM_DecodeU64Numbers(benchmark::State& state) {
592 + std::random_device rd;
593 + std::mt19937 mt(rd());
594 + std::uniform_int_distribution<uint64_t> dist(0x0, 0xFFFFFFFF);
595 +
596 + std::vector<uint64_t> RandomData;
597 + for (size_t idx = 0; idx != 1024; idx++) {
598 + RandomData.push_back(dist(mt));
599 + }
600 + std::vector<uint64_t> EncodedData(10 * RandomData.capacity(), 0);
601 + std::vector<uint64_t> DecodedData(10 * RandomData.capacity(), 0);
602 +
603 + gorilla_encode_u64(EncodedData.data(), EncodedData.size(),
604 + RandomData.data(), RandomData.size());
605 +
606 + for (auto _ : state) {
607 + benchmark::DoNotOptimize(
608 + gorilla_decode_u64(DecodedData.data(), DecodedData.size(),
609 + EncodedData.data(), EncodedData.size())
610 + );
611 + benchmark::ClobberMemory();
612 + }
613 +
614 + state.SetItemsProcessed(NumItems * state.iterations());
615 + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint64_t));
616 +}
617 +// Register the function as a benchmark
618 +BENCHMARK(BM_DecodeU64Numbers);
619 +
620 +#endif /* ENABLE_BENCHMARK */
libnetdata/gorilla/gorilla.h new
+60
@@ -0,0 +1,60 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef GORILLA_H
4 +#define GORILLA_H
5 +
6 +#include <stdbool.h>
7 +#include <stdint.h>
8 +#include <stddef.h>
9 +
10 +#ifdef __cplusplus
11 +extern "C" {
12 +#endif
13 +
14 +/*
15 + * Low-level public API
16 +*/
17 +
18 +// 32-bit API
19 +
20 +typedef struct bit_code_writer_u32 bit_code_writer_u32_t;
21 +typedef struct bit_code_reader_u32 bit_code_reader_u32_t;
22 +
23 +void bit_code_writer_u32_init(bit_code_writer_u32_t *bcw, uint32_t *buffer, uint32_t capacity);
24 +bool bit_code_writer_u32_write(bit_code_writer_u32_t *bcw, const uint32_t number);
25 +bool bit_code_writer_u32_flush(bit_code_writer_u32_t *bcw);
26 +
27 +void bit_code_reader_u32_init(bit_code_reader_u32_t *bcr, uint32_t *buffer, uint32_t capacity);
28 +bool bit_code_reader_u32_read(bit_code_reader_u32_t *bcr, uint32_t *number);
29 +bool bit_code_reader_u32_info(bit_code_reader_u32_t *bcr, uint32_t *num_entries_written,
30 + uint64_t *num_bits_written);
31 +
32 +// 64-bit API
33 +
34 +typedef struct bit_code_writer_u64 bit_code_writer_u64_t;
35 +typedef struct bit_code_reader_u64 bit_code_reader_u64_t;
36 +
37 +void bit_code_writer_u64_init(bit_code_writer_u64_t *bcw, uint64_t *buffer, uint64_t capacity);
38 +bool bit_code_writer_u64_write(bit_code_writer_u64_t *bcw, const uint64_t number);
39 +bool bit_code_writer_u64_flush(bit_code_writer_u64_t *bcw);
40 +
41 +void bit_code_reader_u64_init(bit_code_reader_u64_t *bcr, uint64_t *buffer, uint64_t capacity);
42 +bool bit_code_reader_u64_read(bit_code_reader_u64_t *bcr, uint64_t *number);
43 +bool bit_code_reader_u64_info(bit_code_reader_u64_t *bcr, uint64_t *num_entries_written,
44 + uint64_t *num_bits_written);
45 +
46 +/*
47 + * High-level public API
48 +*/
49 +
50 +size_t gorilla_encode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len);
51 +size_t gorilla_decode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len);
52 +
53 +size_t gorilla_encode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len);
54 +size_t gorilla_decode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len);
55 +
56 +#ifdef __cplusplus
57 +}
58 +#endif
59 +
60 +#endif /* GORILLA_H */
libnetdata/libnetdata.h
+1
@@ -666,6 +666,7 @@ extern char *netdata_configured_host_prefix;
666 #include "parser/parser.h"
667 #include "yaml.h"
668 #include "http/http_defs.h"
669 +#include "gorilla/gorilla.h"
670
671 // BEWARE: this exists in alarm-notify.sh
672 #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"