master
cc 610 lines 18.6 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "libnetdata/common.h"
4 #include "gorilla.h"
5
6 #include <cassert>
7 #include <climits>
8 #include <cstdio>
9 #include <cstring>
10
11 using std::size_t;
12
13 template <typename T>
14 static constexpr size_t bit_size() noexcept
15 {
16 static_assert((sizeof(T) * CHAR_BIT) == 32 || (sizeof(T) * CHAR_BIT) == 64,
17 "Word size should be 32 or 64 bits.");
18 return (sizeof(T) * CHAR_BIT);
19 }
20
21 static uint32_t gorilla_buffer_nbytes(uint32_t nbits) {
22 uint32_t slots = (nbits + RRDENG_GORILLA_32BIT_SLOT_BITS - 1) / RRDENG_GORILLA_32BIT_SLOT_BITS;
23 assert(slots > 0 && slots <= RRDENG_GORILLA_32BIT_BUFFER_SLOTS);
24
25 // this is needed to avoid heap buffer overflow in bit_buffer_read()
26 if(slots < RRDENG_GORILLA_32BIT_BUFFER_SLOTS)
27 slots++;
28
29 return slots * RRDENG_GORILLA_32BIT_SLOT_BYTES;
30 }
31
32 static void bit_buffer_write(uint32_t *buf, size_t pos, uint32_t v, size_t nbits)
33 {
34 assert(nbits > 0 && nbits <= bit_size<uint32_t>());
35
36 const size_t index = pos / bit_size<uint32_t>();
37 const size_t offset = pos % bit_size<uint32_t>();
38
39 pos += nbits;
40
41 if (offset == 0) {
42 buf[index] = v;
43 } else {
44 const size_t remaining_bits = bit_size<uint32_t>() - offset;
45
46 // write the lower part of the value
47 const uint32_t low_bits_mask = ((uint32_t) 1 << remaining_bits) - 1;
48 const uint32_t lowest_bits_in_value = v & low_bits_mask;
49 buf[index] |= (lowest_bits_in_value << offset);
50
51 if (nbits > remaining_bits) {
52 // write the upper part of the value
53 const uint32_t high_bits_mask = ~low_bits_mask;
54 const uint32_t highest_bits_in_value = (v & high_bits_mask) >> (remaining_bits);
55 buf[index + 1] = highest_bits_in_value;
56 }
57 }
58 }
59
60 static void bit_buffer_read(const uint32_t *buf, size_t pos, uint32_t *v, size_t nbits)
61 {
62 assert(nbits > 0 && nbits <= bit_size<uint32_t>());
63
64 const size_t index = pos / bit_size<uint32_t>();
65 const size_t offset = pos % bit_size<uint32_t>();
66
67 pos += nbits;
68
69 if (offset == 0) {
70 *v = (nbits == bit_size<uint32_t>()) ?
71 buf[index] :
72 buf[index] & (((uint32_t) 1 << nbits) - 1);
73 } else {
74 const size_t remaining_bits = bit_size<uint32_t>() - offset;
75
76 // extract the lower part of the value
77 if (nbits < remaining_bits) {
78 *v = (buf[index] >> offset) & (((uint32_t) 1 << nbits) - 1);
79 } else {
80 *v = (buf[index] >> offset) & (((uint32_t) 1 << remaining_bits) - 1);
81 nbits -= remaining_bits;
82 *v |= (buf[index + 1] & (((uint32_t) 1 << nbits) - 1)) << remaining_bits;
83 }
84 }
85 }
86
87 gorilla_writer_t gorilla_writer_init(gorilla_buffer_t *gbuf, size_t n)
88 {
89 gorilla_writer_t gw = gorilla_writer_t {
90 .head_buffer = gbuf,
91 .last_buffer = NULL,
92 .prev_number = 0,
93 .prev_xor_lzc = 0,
94 .capacity = 0
95 };
96
97 gorilla_writer_add_buffer(&gw, gbuf, n);
98 return gw;
99 }
100
101 void gorilla_writer_add_buffer(gorilla_writer_t *gw, gorilla_buffer_t *gbuf, size_t n)
102 {
103 gbuf->header.next = NULL;
104 gbuf->header.entries = 0;
105 gbuf->header.nbits = 0;
106
107 uint32_t capacity = (n * bit_size<uint32_t>()) - (sizeof(gorilla_header_t) * CHAR_BIT);
108
109 gw->prev_number = 0;
110 gw->prev_xor_lzc = 0;
111 gw->capacity = capacity;
112
113 if (gw->last_buffer)
114 gw->last_buffer->header.next = gbuf;
115
116 __atomic_store_n(&gw->last_buffer, gbuf, __ATOMIC_RELEASE);
117 }
118
119 uint32_t gorilla_writer_entries(const gorilla_writer_t *gw) {
120 uint32_t entries = 0;
121
122 const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
123 do {
124 const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
125
126 entries += __atomic_load_n(&curr_gbuf->header.entries, __ATOMIC_ACQUIRE);
127
128 curr_gbuf = next_gbuf;
129 } while (curr_gbuf);
130
131 return entries;
132 }
133
134 extern "C" {
135 ALWAYS_INLINE_ONLY bool gorilla_writer_write(gorilla_writer_t *gw, uint32_t number)
136 {
137 gorilla_header_t *hdr = &gw->last_buffer->header;
138 uint32_t *data = gw->last_buffer->data;
139
140 // this is the first number we are writing
141 if (hdr->entries == 0) {
142 if (hdr->nbits + bit_size<uint32_t>() >= gw->capacity)
143 return false;
144 bit_buffer_write(data, hdr->nbits, number, bit_size<uint32_t>());
145
146 __atomic_fetch_add(&hdr->nbits, bit_size<uint32_t>(), __ATOMIC_RELEASE);
147 __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELEASE);
148 gw->prev_number = number;
149 return true;
150 }
151
152 // write true/false based on whether we got the same number or not.
153 if (number == gw->prev_number) {
154 if (hdr->nbits + 1 >= gw->capacity)
155 return false;
156
157 bit_buffer_write(data, hdr->nbits, static_cast<uint32_t>(1), 1);
158 __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELEASE);
159 __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELEASE);
160 return true;
161 }
162
163 if (hdr->nbits + 1 >= gw->capacity)
164 return false;
165 bit_buffer_write(data, hdr->nbits, static_cast<uint32_t>(0), 1);
166 __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELEASE);
167
168 uint32_t xor_value = gw->prev_number ^ number;
169 uint32_t xor_lzc = (bit_size<uint32_t>() == 32) ? __builtin_clz(xor_value) : __builtin_clzll(xor_value);
170 uint32_t is_xor_lzc_same = (xor_lzc == gw->prev_xor_lzc) ? 1 : 0;
171
172 if (hdr->nbits + 1 >= gw->capacity)
173 return false;
174 bit_buffer_write(data, hdr->nbits, is_xor_lzc_same, 1);
175 __atomic_fetch_add(&hdr->nbits, 1, __ATOMIC_RELEASE);
176
177 if (!is_xor_lzc_same) {
178 size_t bits_needed = (bit_size<uint32_t>() == 32) ? 5 : 6;
179 if ((hdr->nbits + bits_needed) >= gw->capacity)
180 return false;
181 bit_buffer_write(data, hdr->nbits, xor_lzc, bits_needed);
182 __atomic_fetch_add(&hdr->nbits, bits_needed, __ATOMIC_RELEASE);
183 }
184
185 // write the bits of the XOR'd value without the LZC prefix
186 if (hdr->nbits + (bit_size<uint32_t>() - xor_lzc) >= gw->capacity)
187 return false;
188 bit_buffer_write(data, hdr->nbits, xor_value, bit_size<uint32_t>() - xor_lzc);
189 __atomic_fetch_add(&hdr->nbits, bit_size<uint32_t>() - xor_lzc, __ATOMIC_RELEASE);
190 __atomic_fetch_add(&hdr->entries, 1, __ATOMIC_RELEASE);
191
192 gw->prev_number = number;
193 gw->prev_xor_lzc = xor_lzc;
194 return true;
195 }
196 }
197
198 gorilla_buffer_t *gorilla_writer_drop_head_buffer(gorilla_writer_t *gw) {
199 if (!gw->head_buffer)
200 return NULL;
201
202 gorilla_buffer_t *curr_head = gw->head_buffer;
203 gorilla_buffer_t *next_head = gw->head_buffer->header.next;
204 __atomic_store_n(&gw->head_buffer, next_head, __ATOMIC_RELEASE);
205 return curr_head;
206 }
207
208 uint32_t gorilla_writer_actual_nbytes(const gorilla_writer_t *gw)
209 {
210 uint32_t nbytes = 0;
211
212 const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
213 do {
214 const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
215
216 nbytes += RRDENG_GORILLA_32BIT_BUFFER_SIZE;
217
218 curr_gbuf = next_gbuf;
219 } while (curr_gbuf);
220
221 return nbytes;
222 }
223
224 uint32_t gorilla_writer_optimal_nbytes(const gorilla_writer_t *gw)
225 {
226 uint32_t nbytes = 0;
227
228 const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
229 do {
230 const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
231
232 if(next_gbuf)
233 nbytes += RRDENG_GORILLA_32BIT_BUFFER_SIZE;
234 else
235 nbytes += gorilla_buffer_nbytes(__atomic_load_n(&curr_gbuf->header.nbits, __ATOMIC_ACQUIRE));
236
237 curr_gbuf = next_gbuf;
238 } while (curr_gbuf);
239
240 return nbytes;
241 }
242
243 bool gorilla_writer_serialize(const gorilla_writer_t *gw, uint8_t *dst, uint32_t dst_size) {
244 const gorilla_buffer_t *curr_gbuf = gw->head_buffer;
245
246 do {
247 const gorilla_buffer_t *next_gbuf = curr_gbuf->header.next;
248
249 size_t bytes = RRDENG_GORILLA_32BIT_BUFFER_SIZE;
250 if (bytes > dst_size)
251 return false;
252
253 memcpy(dst, curr_gbuf, bytes);
254 dst += bytes;
255 dst_size -= bytes;
256
257 curr_gbuf = next_gbuf;
258 } while (curr_gbuf);
259
260 return true;
261 }
262
263 uint32_t gorilla_buffer_patch(gorilla_buffer_t *gbuf) {
264 gorilla_buffer_t *curr_gbuf = gbuf;
265 uint32_t n = curr_gbuf->header.entries;
266
267 while (curr_gbuf->header.next) {
268 uint32_t *buf = reinterpret_cast<uint32_t *>(gbuf);
269 gbuf = reinterpret_cast<gorilla_buffer_t *>(&buf[RRDENG_GORILLA_32BIT_BUFFER_SLOTS]);
270
271 assert(((uintptr_t) (gbuf) % sizeof(uintptr_t)) == 0 &&
272 "Gorilla buffer not aligned to uintptr_t");
273
274 curr_gbuf->header.next = gbuf;
275 curr_gbuf = curr_gbuf->header.next;
276
277 n += curr_gbuf->header.entries;
278 }
279
280 return n;
281 }
282
283 size_t gorilla_buffer_unpatched_nbuffers(const gorilla_buffer_t *gbuf) {
284 size_t nbuffers = 0;
285 while(gbuf) {
286 nbuffers++;
287
288 if(gbuf->header.next) {
289 const auto *buf = reinterpret_cast<const uint32_t *>(gbuf);
290 gbuf = reinterpret_cast<const gorilla_buffer_t *>(&buf[RRDENG_GORILLA_32BIT_BUFFER_SLOTS]);
291 }
292 else
293 break;
294 }
295
296 return nbuffers;
297 }
298
299 size_t gorilla_buffer_unpatched_nbytes(const gorilla_buffer_t *gbuf) {
300 size_t nbytes = sizeof(gorilla_buffer_t);
301 while(gbuf) {
302 if(gbuf->header.next) {
303 nbytes += RRDENG_GORILLA_32BIT_BUFFER_SIZE;
304 const auto *buf = reinterpret_cast<const uint32_t *>(gbuf);
305 gbuf = reinterpret_cast<const gorilla_buffer_t *>(&buf[RRDENG_GORILLA_32BIT_BUFFER_SLOTS]);
306 }
307 else {
308 nbytes += gorilla_buffer_nbytes(gbuf->header.nbits);
309 break;
310 }
311 }
312
313 return nbytes;
314 }
315
316 gorilla_reader_t gorilla_writer_get_reader(const gorilla_writer_t *gw)
317 {
318 const gorilla_buffer_t *buffer = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
319
320 uint32_t entries = __atomic_load_n(&buffer->header.entries, __ATOMIC_ACQUIRE);
321 uint32_t capacity = __atomic_load_n(&buffer->header.nbits, __ATOMIC_ACQUIRE);
322
323 return gorilla_reader_t {
324 .buffer = buffer,
325 .entries = entries,
326 .index = 0,
327 .capacity = capacity,
328 .position = 0,
329 .prev_number = 0,
330 .prev_xor_lzc = 0,
331 .prev_xor = 0,
332 };
333 }
334
335 gorilla_reader_t gorilla_reader_init(gorilla_buffer_t *gbuf)
336 {
337 uint32_t entries = __atomic_load_n(&gbuf->header.entries, __ATOMIC_ACQUIRE);
338 uint32_t capacity = __atomic_load_n(&gbuf->header.nbits, __ATOMIC_ACQUIRE);
339
340 return gorilla_reader_t {
341 .buffer = gbuf,
342 .entries = entries,
343 .index = 0,
344 .capacity = capacity,
345 .position = 0,
346 .prev_number = 0,
347 .prev_xor_lzc = 0,
348 .prev_xor = 0,
349 };
350 }
351
352 extern "C" {
353 ALWAYS_INLINE_ONLY bool gorilla_reader_read(gorilla_reader_t *gr, uint32_t *number)
354 {
355 const uint32_t *data = gr->buffer->data;
356
357 while (gr->index + 1 > gr->entries) {
358 // We don't have any more entries to return. However, the writer
359 // might have updated the buffer's entries. We need to check once
360 // more in case more elements were added.
361 gr->entries = __atomic_load_n(&gr->buffer->header.entries, __ATOMIC_ACQUIRE);
362 gr->capacity = __atomic_load_n(&gr->buffer->header.nbits, __ATOMIC_ACQUIRE);
363
364 // if the reader's current buffer has not been updated, we need to
365 // check if it has a pointer to a next buffer.
366 if (gr->index + 1 > gr->entries) {
367 gorilla_buffer_t *next_buffer = __atomic_load_n(&gr->buffer->header.next, __ATOMIC_ACQUIRE);
368
369 if (!next_buffer) {
370 // fprintf(stderr, "Consumed reader with %zu entries from buffer %p\n (No more buffers to read from)", gr->length, gr->buffer);
371 return false;
372 }
373
374 // fprintf(stderr, "Consumed reader with %zu entries from buffer %p\n", gr->length, gr->buffer);
375 *gr = gorilla_reader_init(next_buffer);
376 data = gr->buffer->data;
377 }
378 else
379 break;
380 }
381
382 // read the first number
383 if (gr->index == 0) {
384 bit_buffer_read(data, gr->position, number, bit_size<uint32_t>());
385
386 gr->index++;
387 gr->position += bit_size<uint32_t>();
388 gr->prev_number = *number;
389 return true;
390 }
391
392 // process same-number bit
393 uint32_t is_same_number;
394 bit_buffer_read(data, gr->position, &is_same_number, 1);
395 gr->position++;
396
397 if (is_same_number) {
398 *number = gr->prev_number;
399 gr->index++;
400 return true;
401 }
402
403 // proceess same-xor-lzc bit
404 uint32_t xor_lzc = gr->prev_xor_lzc;
405
406 uint32_t same_xor_lzc;
407 bit_buffer_read(data, gr->position, &same_xor_lzc, 1);
408 gr->position++;
409
410 if (!same_xor_lzc) {
411 bit_buffer_read(data, gr->position, &xor_lzc, (bit_size<uint32_t>() == 32) ? 5 : 6);
412 gr->position += (bit_size<uint32_t>() == 32) ? 5 : 6;
413 }
414
415 // process the non-lzc suffix
416 uint32_t xor_value = 0;
417 bit_buffer_read(data, gr->position, &xor_value, bit_size<uint32_t>() - xor_lzc);
418 gr->position += bit_size<uint32_t>() - xor_lzc;
419
420 *number = (gr->prev_number ^ xor_value);
421
422 gr->index++;
423 gr->prev_number = *number;
424 gr->prev_xor_lzc = xor_lzc;
425 gr->prev_xor = xor_value;
426
427 return true;
428 }
429 }
430
431 extern "C" {
432 struct aral;
433 void aral_unmark_allocation(struct aral *ar, void *ptr);
434 }
435
436 void gorilla_writer_aral_unmark(const gorilla_writer_t *gw, struct aral *ar)
437 {
438 const gorilla_buffer_t *curr_gbuf = __atomic_load_n(&gw->head_buffer, __ATOMIC_ACQUIRE);
439 while (curr_gbuf) {
440 const gorilla_buffer_t *next_gbuf = __atomic_load_n(&curr_gbuf->header.next, __ATOMIC_ACQUIRE);
441
442 aral_unmark_allocation(ar, const_cast<void*>(static_cast<const void*>(curr_gbuf)));
443
444 curr_gbuf = next_gbuf;
445 }
446 }
447
448 /*
449 * Internal code used for fuzzing the library
450 */
451
452 #ifdef ENABLE_FUZZER
453
454 #include <vector>
455
456 template<typename Word>
457 static std::vector<Word> random_vector(const uint8_t *data, size_t size) {
458 std::vector<Word> V;
459
460 V.reserve(1024);
461
462 while (size >= sizeof(Word)) {
463 size -= sizeof(Word);
464
465 Word w;
466 memcpy(&w, &data[size], sizeof(Word));
467 V.push_back(w);
468 }
469
470 return V;
471 }
472
473 class Storage {
474 public:
475 gorilla_buffer_t *alloc_buffer(size_t words) {
476 uint32_t *new_buffer = new uint32_t[words]();
477 assert(((((uintptr_t) new_buffer) % 8u) == 0) && "Unaligned buffer...");
478 Buffers.push_back(new_buffer);
479 return reinterpret_cast<gorilla_buffer_t *>(new_buffer);
480 }
481
482 void free_buffers() {
483 for (uint32_t *buffer : Buffers) {
484 delete[] buffer;
485 }
486 }
487
488 private:
489 std::vector<uint32_t *> Buffers;
490 };
491
492 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
493 if (Size < 4)
494 return 0;
495
496 std::vector<uint32_t> RandomData = random_vector<uint32_t>(Data, Size);
497
498 Storage S;
499 size_t words_per_buffer = 8;
500
501 /*
502 * write data
503 */
504 gorilla_buffer_t *first_buffer = S.alloc_buffer(words_per_buffer);
505 gorilla_writer_t gw = gorilla_writer_init(first_buffer, words_per_buffer);
506
507 for (size_t i = 0; i != RandomData.size(); i++) {
508 bool ok = gorilla_writer_write(&gw, RandomData[i]);
509 if (ok)
510 continue;
511
512 // add new buffer
513 gorilla_buffer_t *buffer = S.alloc_buffer(words_per_buffer);
514 gorilla_writer_add_buffer(&gw, buffer, words_per_buffer);
515
516 ok = gorilla_writer_write(&gw, RandomData[i]);
517 assert(ok && "Could not write data to new buffer!!!");
518 }
519
520
521 /*
522 * read data
523 */
524 gorilla_reader_t gr = gorilla_writer_get_reader(&gw);
525
526 for (size_t i = 0; i != RandomData.size(); i++) {
527 uint32_t number = 0;
528 bool ok = gorilla_reader_read(&gr, &number);
529 assert(ok && "Failed to read number from gorilla buffer");
530
531 assert((number == RandomData[i])
532 && "Read wrong number from gorilla buffer");
533 }
534
535 S.free_buffers();
536 return 0;
537 }
538
539 #endif /* ENABLE_FUZZER */
540
541 #ifdef ENABLE_BENCHMARK
542
543 #include <benchmark/benchmark.h>
544 #include <random>
545
546 static size_t NumItems = 1024;
547
548 static void BM_EncodeU32Numbers(benchmark::State& state) {
549 std::random_device rd;
550 std::mt19937 mt(rd());
551 std::uniform_int_distribution<uint32_t> dist(0x0, 0x0000FFFF);
552
553 std::vector<uint32_t> RandomData;
554 for (size_t idx = 0; idx != NumItems; idx++) {
555 RandomData.push_back(dist(mt));
556 }
557 std::vector<uint32_t> EncodedData(10 * RandomData.capacity(), 0);
558
559 for (auto _ : state) {
560 gorilla_writer_t gw = gorilla_writer_init(
561 reinterpret_cast<gorilla_buffer_t *>(EncodedData.data()),
562 EncodedData.size());
563
564 for (size_t i = 0; i != RandomData.size(); i++)
565 benchmark::DoNotOptimize(gorilla_writer_write(&gw, RandomData[i]));
566
567 benchmark::ClobberMemory();
568 }
569
570 state.SetItemsProcessed(NumItems * state.iterations());
571 state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint32_t));
572 }
573 BENCHMARK(BM_EncodeU32Numbers)->ThreadRange(1, 16)->UseRealTime();
574
575 static void BM_DecodeU32Numbers(benchmark::State& state) {
576 std::random_device rd;
577 std::mt19937 mt(rd());
578 std::uniform_int_distribution<uint32_t> dist(0x0, 0xFFFFFFFF);
579
580 std::vector<uint32_t> RandomData;
581 for (size_t idx = 0; idx != NumItems; idx++) {
582 RandomData.push_back(dist(mt));
583 }
584 std::vector<uint32_t> EncodedData(10 * RandomData.capacity(), 0);
585 std::vector<uint32_t> DecodedData(10 * RandomData.capacity(), 0);
586
587 gorilla_writer_t gw = gorilla_writer_init(
588 reinterpret_cast<gorilla_buffer_t *>(EncodedData.data()),
589 EncodedData.size());
590
591 for (size_t i = 0; i != RandomData.size(); i++)
592 gorilla_writer_write(&gw, RandomData[i]);
593
594 for (auto _ : state) {
595 gorilla_reader_t gr = gorilla_reader_init(reinterpret_cast<gorilla_buffer_t *>(EncodedData.data()));
596
597 for (size_t i = 0; i != RandomData.size(); i++) {
598 uint32_t number = 0;
599 benchmark::DoNotOptimize(gorilla_reader_read(&gr, &number));
600 }
601
602 benchmark::ClobberMemory();
603 }
604
605 state.SetItemsProcessed(NumItems * state.iterations());
606 state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint32_t));
607 }
608 BENCHMARK(BM_DecodeU32Numbers)->ThreadRange(1, 16)->UseRealTime();
609
610 #endif /* ENABLE_BENCHMARK */