Code cleanup (#17237)
* renames in dbengine * remove leftovers from memory mode save and map * fix docs about 3 tiers by default * split linked-lists, bitmaps and storage-points from libnetdata.h
Costa Tsaousis committed
Mar 24, 2024 at 00:55 UTC
00f897a883bde68ba3df587655a30225fbc4ecdd
19 files changed
+447
-434
CMakeLists.txt
+3
@@ -673,6 +673,9 @@ set(LIBNETDATA_FILES
673
src/libnetdata/dictionary/dictionary-hashtable.h
674
src/libnetdata/dictionary/dictionary-item.h
675
src/libnetdata/dictionary/dictionary-callbacks.h
676
+ src/libnetdata/linked-lists.h
677
+ src/libnetdata/storage-point.h
678
+ src/libnetdata/bitmap.h
679
)
680
681
if(ENABLE_PLUGIN_EBPF)
src/collectors/cgroups.plugin/README.md
-2
@@ -270,8 +270,6 @@ a few errors in error.log complaining about files it cannot find, but immediatel
270
5. Existing dashboard sessions will continue to see them, but of course they will not refresh
271
6. Obsolete charts will be removed from memory, 1 hour after the last user viewed them (configurable
272
with `[global].cleanup obsolete charts after seconds = 3600` (at `netdata.conf`).
273
-7. When obsolete charts are removed from memory they are also deleted from disk (configurable
274
- with `[global].delete obsolete charts files = yes`)
273
274
### Monitored container metrics
275
src/daemon/config/README.md
+1
-3
@@ -90,7 +90,7 @@ Please note that your data history will be lost if you have modified `history` p
90
|:---------------------------------------------:|:----------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
91
| mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `dbengine page cache size MB` and `dbengine disk space MB`. <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`alloc`: Similar to `ram`, but can significantly reduce memory usage, when combined with a low retention and does not support KSM. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. Not to be used together with streaming. |
92
| retention | `3600` | Used with `mode = ram/alloc`, not the default `mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. Check [Memory Requirements](https://github.com/netdata/netdata/blob/master/src/database/README.md) for more information. |
93
-| storage tiers | `1` | The number of storage tiers you want to have in your dbengine. Check the tiering mechanism in the [dbengine's reference](https://github.com/netdata/netdata/blob/master/src/database/engine/README.md#tiering). You can have up to 5 tiers of data (including the _Tier 0_). This number ranges between 1 and 5. |
93
+| storage tiers | `3` | The number of storage tiers you want to have in your dbengine. Check the tiering mechanism in the [dbengine's reference](https://github.com/netdata/netdata/blob/master/src/database/engine/README.md#tiering). You can have up to 5 tiers of data (including the _Tier 0_). This number ranges between 1 and 5. |
94
| dbengine page cache size MB | `32` | Determines the amount of RAM in MiB that is dedicated to caching for _Tier 0_ Netdata metric values. |
95
| dbengine tier **`N`** page cache size MB | `32` | Determines the amount of RAM in MiB that is dedicated for caching Netdata metric values of the **`N`** tier. <br /> `N belongs to [1..4]` |
96
| dbengine disk space MB | `256` | Determines the amount of disk space in MiB that is dedicated to storing _Tier 0_ Netdata metric values and all related metadata describing them. This option is available **only for legacy configuration** (`Agent v1.23.2 and prior`). |
@@ -103,8 +103,6 @@ Please note that your data history will be lost if you have modified `history` p
103
| cleanup obsolete charts after secs | `3600` | See [monitoring ephemeral containers](https://github.com/netdata/netdata/blob/master/src/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions |
104
| gap when lost iterations above | `1` | |
105
| cleanup orphan hosts after secs | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. |
106
-| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](https://github.com/netdata/netdata/blob/master/src/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions |
107
-| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. |
106
| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. |
107
108
> ### Info
src/daemon/global_statistics.c
+2
-2
@@ -858,7 +858,7 @@ static void global_statistics_charts(void) {
858
// ----------------------------------------------------------------
859
860
#ifdef ENABLE_DBENGINE
861
- if (tier_page_type[0] == PAGE_GORILLA_METRICS)
861
+ if (tier_page_type[0] == RRDENG_PAGE_TYPE_GORILLA_32BIT)
862
{
863
static RRDSET *st_tier0_gorilla_pages = NULL;
864
static RRDDIM *rd_num_gorilla_pages = NULL;
@@ -887,7 +887,7 @@ static void global_statistics_charts(void) {
887
rrdset_done(st_tier0_gorilla_pages);
888
}
889
890
- if (tier_page_type[0] == PAGE_GORILLA_METRICS)
890
+ if (tier_page_type[0] == RRDENG_PAGE_TYPE_GORILLA_32BIT)
891
{
892
static RRDSET *st_tier0_compression_info = NULL;
893
src/daemon/main.c
+3
-9
@@ -1054,12 +1054,6 @@ static void backwards_compatible_config() {
1054
config_move(CONFIG_SECTION_GLOBAL, "cleanup orphan hosts after seconds",
1055
CONFIG_SECTION_DB, "cleanup orphan hosts after secs");
1056
1057
- config_move(CONFIG_SECTION_GLOBAL, "delete obsolete charts files",
1058
- CONFIG_SECTION_DB, "delete obsolete charts files");
1059
-
1060
- config_move(CONFIG_SECTION_GLOBAL, "delete orphan hosts files",
1061
- CONFIG_SECTION_DB, "delete orphan hosts files");
1062
-
1057
config_move(CONFIG_SECTION_GLOBAL, "enable zero metrics",
1058
CONFIG_SECTION_DB, "enable zero metrics");
1059
@@ -1171,11 +1165,11 @@ static void get_netdata_configured_variables() {
1165
1166
const char *page_type = config_get(CONFIG_SECTION_DB, "dbengine page type", "gorilla");
1167
if (strcmp(page_type, "gorilla") == 0)
1174
- tier_page_type[0] = PAGE_GORILLA_METRICS;
1168
+ tier_page_type[0] = RRDENG_PAGE_TYPE_GORILLA_32BIT;
1169
else if (strcmp(page_type, "raw") == 0)
1176
- tier_page_type[0] = PAGE_METRICS;
1170
+ tier_page_type[0] = RRDENG_PAGE_TYPE_ARRAY_32BIT;
1171
else {
1178
- tier_page_type[0] = PAGE_METRICS;
1172
+ tier_page_type[0] = RRDENG_PAGE_TYPE_ARRAY_32BIT;
1173
netdata_log_error("Invalid dbengine page type ''%s' given. Defaulting to 'raw'.", page_type);
1174
}
1175
src/database/engine/journalfile.c
+1
-1
@@ -672,7 +672,7 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
672
uuid_t *temp_id;
673
uint8_t page_type = jf_metric_data->descr[i].type;
674
675
- if (page_type > PAGE_TYPE_MAX) {
675
+ if (page_type > RRDENG_PAGE_TYPE_MAX) {
676
if (!bitmap256_get_bit(&page_error_map, page_type)) {
677
netdata_log_error("DBENGINE: unknown page type %d encountered.", page_type);
678
bitmap256_set_bit(&page_error_map, page_type, 1);
src/database/engine/page.c
+38
-37
@@ -111,9 +111,9 @@ void pgd_init_arals(void)
111
// FIXME: add stats
112
pgd_alloc_globals.aral_gorilla_buffer[i] = aral_create(
113
buf,
114
- GORILLA_BUFFER_SIZE,
114
+ RRDENG_GORILLA_32BIT_BUFFER_SIZE,
115
64,
116
- 512 * GORILLA_BUFFER_SIZE,
116
+ 512 * RRDENG_GORILLA_32BIT_BUFFER_SIZE,
117
pgc_aral_statistics(),
118
NULL, NULL, false, false);
119
}
@@ -165,8 +165,8 @@ PGD *pgd_create(uint8_t type, uint32_t slots)
165
pg->states = PGD_STATE_CREATED_FROM_COLLECTOR;
166
167
switch (type) {
168
- case PAGE_METRICS:
169
- case PAGE_TIER: {
168
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
169
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
170
uint32_t size = slots * page_type_size[type];
171
172
internal_fatal(!size || slots == 1,
@@ -176,11 +176,11 @@ PGD *pgd_create(uint8_t type, uint32_t slots)
176
pg->raw.data = pgd_data_aral_alloc(size);
177
break;
178
}
179
- case PAGE_GORILLA_METRICS: {
179
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
180
internal_fatal(slots == 1,
181
"DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
182
183
- pg->slots = 8 * GORILLA_BUFFER_SLOTS;
183
+ pg->slots = 8 * RRDENG_GORILLA_32BIT_BUFFER_SLOTS;
184
185
// allocate new gorilla writer
186
pg->gorilla.aral_index = gettid() % 4;
@@ -188,10 +188,10 @@ PGD *pgd_create(uint8_t type, uint32_t slots)
188
189
// allocate new gorilla buffer
190
gorilla_buffer_t *gbuf = aral_mallocz(pgd_alloc_globals.aral_gorilla_buffer[pg->gorilla.aral_index]);
191
- memset(gbuf, 0, GORILLA_BUFFER_SIZE);
191
+ memset(gbuf, 0, RRDENG_GORILLA_32BIT_BUFFER_SIZE);
192
global_statistics_gorilla_buffer_add_hot();
193
194
- *pg->gorilla.writer = gorilla_writer_init(gbuf, GORILLA_BUFFER_SLOTS);
194
+ *pg->gorilla.writer = gorilla_writer_init(gbuf, RRDENG_GORILLA_32BIT_BUFFER_SLOTS);
195
pg->gorilla.num_buffers = 1;
196
197
break;
@@ -222,8 +222,8 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size)
222
223
switch (type)
224
{
225
- case PAGE_METRICS:
226
- case PAGE_TIER:
225
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
226
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
227
pg->raw.size = size;
228
pg->used = size / page_type_size[type];
229
pg->slots = pg->used;
@@ -231,10 +231,11 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size)
231
pg->raw.data = pgd_data_aral_alloc(size);
232
memcpy(pg->raw.data, base, size);
233
break;
234
- case PAGE_GORILLA_METRICS:
234
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT:
235
internal_fatal(size == 0, "Asked to create page with 0 data!!!");
236
internal_fatal(size % sizeof(uint32_t), "Unaligned gorilla buffer size");
237
- internal_fatal(size % GORILLA_BUFFER_SIZE, "Expected size to be a multiple of %zu-bytes", GORILLA_BUFFER_SIZE);
237
+ internal_fatal(size % RRDENG_GORILLA_32BIT_BUFFER_SIZE, "Expected size to be a multiple of %zu-bytes",
238
+ RRDENG_GORILLA_32BIT_BUFFER_SIZE);
239
240
pg->raw.data = mallocz(size);
241
pg->raw.size = size;
@@ -268,11 +269,11 @@ void pgd_free(PGD *pg)
269
270
switch (pg->type)
271
{
271
- case PAGE_METRICS:
272
- case PAGE_TIER:
272
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
273
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
274
pgd_data_aral_free(pg->raw.data, pg->raw.size);
275
break;
275
- case PAGE_GORILLA_METRICS: {
276
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
277
if (pg->states & PGD_STATE_CREATED_FROM_DISK)
278
{
279
internal_fatal(pg->raw.data == NULL, "Tried to free gorilla PGD loaded from disk with NULL data");
@@ -365,15 +366,15 @@ uint32_t pgd_memory_footprint(PGD *pg)
366
367
size_t footprint = 0;
368
switch (pg->type) {
368
- case PAGE_METRICS:
369
- case PAGE_TIER:
369
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
370
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
371
footprint = sizeof(PGD) + pg->raw.size;
372
break;
372
- case PAGE_GORILLA_METRICS: {
373
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
374
if (pg->states & PGD_STATE_CREATED_FROM_DISK)
375
footprint = sizeof(PGD) + pg->raw.size;
376
else
376
- footprint = sizeof(PGD) + sizeof(gorilla_writer_t) + (pg->gorilla.num_buffers * GORILLA_BUFFER_SIZE);
377
+ footprint = sizeof(PGD) + sizeof(gorilla_writer_t) + (pg->gorilla.num_buffers * RRDENG_GORILLA_32BIT_BUFFER_SIZE);
378
379
break;
380
}
@@ -393,15 +394,15 @@ uint32_t pgd_disk_footprint(PGD *pg)
394
size_t size = 0;
395
396
switch (pg->type) {
396
- case PAGE_METRICS:
397
- case PAGE_TIER: {
397
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
398
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
399
uint32_t used_size = pg->used * page_type_size[pg->type];
400
internal_fatal(used_size > pg->raw.size, "Wrong disk footprint page size");
401
size = used_size;
402
403
break;
404
}
404
- case PAGE_GORILLA_METRICS: {
405
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
406
if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR ||
407
pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING ||
408
pg->states & PGD_STATE_FLUSHED_TO_DISK)
@@ -412,7 +413,7 @@ uint32_t pgd_disk_footprint(PGD *pg)
413
internal_fatal(pg->gorilla.num_buffers == 0,
414
"Gorilla writer does not have any buffers");
415
415
- size = pg->gorilla.num_buffers * GORILLA_BUFFER_SIZE;
416
+ size = pg->gorilla.num_buffers * RRDENG_GORILLA_32BIT_BUFFER_SIZE;
417
418
if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR) {
419
global_statistics_tier0_disk_compressed_bytes(gorilla_writer_nbytes(pg->gorilla.writer));
@@ -443,11 +444,11 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size)
444
pgd_disk_footprint(pg), dst_size);
445
446
switch (pg->type) {
446
- case PAGE_METRICS:
447
- case PAGE_TIER:
447
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
448
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
449
memcpy(dst, pg->raw.data, dst_size);
450
break;
450
- case PAGE_GORILLA_METRICS: {
451
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
452
if ((pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) == 0)
453
fatal("Copying to extent is supported only for PGDs that are scheduled for flushing.");
454
@@ -500,7 +501,7 @@ void pgd_append_point(PGD *pg,
501
fatal("Data collection on page already scheduled for flushing");
502
503
switch (pg->type) {
503
- case PAGE_METRICS: {
504
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT: {
505
storage_number *tier0_metric_data = (storage_number *)pg->raw.data;
506
storage_number t = pack_storage_number(n, flags);
507
tier0_metric_data[pg->used++] = t;
@@ -510,7 +511,7 @@ void pgd_append_point(PGD *pg,
511
512
break;
513
}
513
- case PAGE_TIER: {
514
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
515
storage_number_tier1_t *tier12_metric_data = (storage_number_tier1_t *)pg->raw.data;
516
storage_number_tier1_t t;
517
t.sum_value = (float) n;
@@ -525,7 +526,7 @@ void pgd_append_point(PGD *pg,
526
527
break;
528
}
528
- case PAGE_GORILLA_METRICS: {
529
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
530
pg->used++;
531
storage_number t = pack_storage_number(n, flags);
532
@@ -535,9 +536,9 @@ void pgd_append_point(PGD *pg,
536
bool ok = gorilla_writer_write(pg->gorilla.writer, t);
537
if (!ok) {
538
gorilla_buffer_t *new_buffer = aral_mallocz(pgd_alloc_globals.aral_gorilla_buffer[pg->gorilla.aral_index]);
538
- memset(new_buffer, 0, GORILLA_BUFFER_SIZE);
539
+ memset(new_buffer, 0, RRDENG_GORILLA_32BIT_BUFFER_SIZE);
540
540
- gorilla_writer_add_buffer(pg->gorilla.writer, new_buffer, GORILLA_BUFFER_SLOTS);
541
+ gorilla_writer_add_buffer(pg->gorilla.writer, new_buffer, RRDENG_GORILLA_32BIT_BUFFER_SLOTS);
542
pg->gorilla.num_buffers += 1;
543
global_statistics_gorilla_buffer_add_hot();
544
@@ -560,11 +561,11 @@ static void pgdc_seek(PGDC *pgdc, uint32_t position)
561
PGD *pg = pgdc->pgd;
562
563
switch (pg->type) {
563
- case PAGE_METRICS:
564
- case PAGE_TIER:
564
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
565
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
566
pgdc->slots = pgdc->pgd->used;
567
break;
567
- case PAGE_GORILLA_METRICS: {
568
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
569
if (pg->states & PGD_STATE_CREATED_FROM_DISK) {
570
pgdc->slots = pgdc->pgd->slots;
571
pgdc->gr = gorilla_reader_init((void *) pg->raw.data);
@@ -634,7 +635,7 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
635
636
switch (pgdc->pgd->type)
637
{
637
- case PAGE_METRICS: {
638
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT: {
639
storage_number *array = (storage_number *) pgdc->pgd->raw.data;
640
storage_number n = array[pgdc->position++];
641
@@ -645,7 +646,7 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
646
647
return true;
648
}
648
- case PAGE_TIER: {
649
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1: {
650
storage_number_tier1_t *array = (storage_number_tier1_t *) pgdc->pgd->raw.data;
651
storage_number_tier1_t n = array[pgdc->position++];
652
@@ -658,7 +659,7 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
659
660
return true;
661
}
661
- case PAGE_GORILLA_METRICS: {
662
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT: {
663
pgdc->position++;
664
665
uint32_t n = 666666666;
src/database/engine/pdc.c
+18
-17
@@ -635,12 +635,12 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_
635
size_t entries = 0;
636
637
switch (descr->type) {
638
- case PAGE_METRICS:
639
- case PAGE_TIER:
638
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
639
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
640
end_time_s = descr->end_time_ut / USEC_PER_SEC;
641
entries = 0;
642
break;
643
- case PAGE_GORILLA_METRICS:
643
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT:
644
end_time_s = start_time_s + descr->gorilla.delta_time_s;
645
entries = descr->gorilla.entries;
646
break;
@@ -689,8 +689,8 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(
689
690
bool known_page_type = true;
691
switch (page_type) {
692
- case PAGE_METRICS:
693
- case PAGE_TIER:
692
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
693
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
694
// always calculate entries by size
695
vd.entries = page_entries_by_size(vd.page_length, vd.point_size);
696
@@ -698,7 +698,7 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(
698
if(!entries)
699
entries = vd.entries;
700
break;
701
- case PAGE_GORILLA_METRICS:
701
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT:
702
internal_fatal(entries == 0, "0 number of entries found on gorilla page");
703
vd.entries = entries;
704
break;
@@ -725,7 +725,7 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(
725
// If gorilla can not compress the data we might end up needing slightly more
726
// than 4KiB. However, gorilla pages extend the page length by increments of
727
// 512 bytes.
728
- max_page_length += ((page_type == PAGE_GORILLA_METRICS) * GORILLA_BUFFER_SIZE);
728
+ max_page_length += ((page_type == RRDENG_PAGE_TYPE_GORILLA_32BIT) * RRDENG_GORILLA_32BIT_BUFFER_SIZE);
729
730
if (!known_page_type ||
731
have_read_error ||
@@ -873,11 +873,11 @@ static void epdl_extent_loading_error_log(struct rrdengine_instance *ctx, EPDL *
873
if (descr) {
874
start_time_s = (time_t)(descr->start_time_ut / USEC_PER_SEC);
875
switch (descr->type) {
876
- case PAGE_METRICS:
877
- case PAGE_TIER:
876
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
877
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
878
end_time_s = (time_t)(descr->end_time_ut / USEC_PER_SEC);
879
break;
880
- case PAGE_GORILLA_METRICS:
880
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT:
881
end_time_s = (time_t) start_time_s + (descr->gorilla.delta_time_s);
882
break;
883
}
@@ -975,10 +975,10 @@ static bool epdl_populate_pages_from_extent_data(
975
if( !can_use_data ||
976
count < 1 ||
977
count > MAX_PAGES_PER_EXTENT ||
978
- (header->compression_algorithm != RRD_NO_COMPRESSION && header->compression_algorithm != RRD_LZ4) ||
978
+ (header->compression_algorithm != RRDENG_COMPRESSION_NONE && header->compression_algorithm != RRDENG_COMPRESSION_LZ4) ||
979
(payload_length != trailer_offset - payload_offset) ||
980
(data_length != payload_offset + payload_length + sizeof(*trailer))
981
- ) {
981
+ ) {
982
epdl_extent_loading_error_log(ctx, epdl, NULL, "header is INVALID");
983
return false;
984
}
@@ -995,14 +995,15 @@ static bool epdl_populate_pages_from_extent_data(
995
if(worker)
996
worker_is_busy(UV_EVENT_DBENGINE_EXTENT_DECOMPRESSION);
997
998
- if (likely(!have_read_error && RRD_NO_COMPRESSION != header->compression_algorithm)) {
998
+ if (likely(!have_read_error && RRDENG_COMPRESSION_NONE != header->compression_algorithm)) {
999
// find the uncompressed extent size
1000
uncompressed_payload_length = 0;
1001
for (i = 0; i < count; ++i) {
1002
size_t page_length = header->descr[i].page_length;
1003
- if (page_length > RRDENG_BLOCK_SIZE && (header->descr[i].type != PAGE_GORILLA_METRICS ||
1004
- (header->descr[i].type == PAGE_GORILLA_METRICS &&
1005
- (page_length - RRDENG_BLOCK_SIZE) % GORILLA_BUFFER_SIZE))) {
1003
+ if (page_length > RRDENG_BLOCK_SIZE &&
1004
+ (header->descr[i].type != RRDENG_PAGE_TYPE_GORILLA_32BIT ||
1005
+ (header->descr[i].type == RRDENG_PAGE_TYPE_GORILLA_32BIT &&
1006
+ (page_length - RRDENG_BLOCK_SIZE) % RRDENG_GORILLA_32BIT_BUFFER_SIZE))) {
1007
have_read_error = true;
1008
break;
1009
}
@@ -1077,7 +1078,7 @@ static bool epdl_populate_pages_from_extent_data(
1078
stats_load_invalid_page++;
1079
}
1080
else {
1080
- if (RRD_NO_COMPRESSION == header->compression_algorithm) {
1081
+ if (RRDENG_COMPRESSION_NONE == header->compression_algorithm) {
1082
pgd = pgd_create_from_disk_data(header->descr[i].type,
1083
data + payload_offset + page_offset,
1084
vd.page_length);
src/database/engine/rrddiskprotocol.h
+9
-6
@@ -19,13 +19,15 @@
19
#define UUID_SZ (16)
20
#define CHECKSUM_SZ (4) /* CRC32 */
21
22
-#define RRD_NO_COMPRESSION (0)
23
-#define RRD_LZ4 (1)
22
+#define RRDENG_COMPRESSION_NONE (0)
23
+#define RRDENG_COMPRESSION_LZ4 (1)
24
25
#define RRDENG_DF_SB_PADDING_SZ (RRDENG_BLOCK_SIZE - (RRDENG_MAGIC_SZ + RRDENG_VER_SZ + sizeof(uint8_t)))
26
+
27
/*
28
* Data file persistent super-block
29
*/
30
+
31
struct rrdeng_df_sb {
32
char magic_number[RRDENG_MAGIC_SZ];
33
char version[RRDENG_VER_SZ];
@@ -36,10 +38,11 @@ struct rrdeng_df_sb {
38
/*
39
* Page types
40
*/
39
-#define PAGE_METRICS (0)
40
-#define PAGE_TIER (1)
41
-#define PAGE_GORILLA_METRICS (2)
42
-#define PAGE_TYPE_MAX 2 // Maximum page type (inclusive)
41
+
42
+#define RRDENG_PAGE_TYPE_ARRAY_32BIT (0)
43
+#define RRDENG_PAGE_TYPE_ARRAY_TIER1 (1)
44
+#define RRDENG_PAGE_TYPE_GORILLA_32BIT (2)
45
+#define RRDENG_PAGE_TYPE_MAX (2) // Maximum page type (inclusive)
46
47
/*
48
* Data file page descriptor
src/database/engine/rrdengine.c
+5
-5
@@ -808,7 +808,7 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta
808
xt_io_descr->ctx = ctx;
809
payload_offset = sizeof(*header) + count * sizeof(header->descr[0]);
810
switch (compression_algorithm) {
811
- case RRD_NO_COMPRESSION:
811
+ case RRDENG_COMPRESSION_NONE:
812
size_bytes = payload_offset + uncompressed_payload_length + sizeof(*trailer);
813
break;
814
@@ -844,11 +844,11 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta
844
header->descr[i].start_time_ut = descr->start_time_ut;
845
846
switch (descr->type) {
847
- case PAGE_METRICS:
848
- case PAGE_TIER:
847
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
848
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
849
header->descr[i].end_time_ut = descr->end_time_ut;
850
break;
851
- case PAGE_GORILLA_METRICS:
851
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT:
852
header->descr[i].gorilla.delta_time_s = (uint32_t) ((descr->end_time_ut - descr->start_time_ut) / USEC_PER_SEC);
853
header->descr[i].gorilla.entries = pgd_slots_used(descr->pgd);
854
break;
@@ -864,7 +864,7 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta
864
pos += descr->page_length;
865
}
866
867
- if(likely(compression_algorithm == RRD_LZ4)) {
867
+ if(likely(compression_algorithm == RRDENG_COMPRESSION_LZ4)) {
868
compressed_size = LZ4_compress_default(
869
xt_io_descr->buf + payload_offset,
870
compressed_buf,
src/database/engine/rrdengineapi.c
+15
-10
@@ -16,7 +16,12 @@ struct rrdengine_instance multidb_ctx_storage_tier4;
16
#error RRD_STORAGE_TIERS is not 5 - you need to add allocations here
17
#endif
18
struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
19
-uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_GORILLA_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER};
19
+uint8_t tier_page_type[RRD_STORAGE_TIERS] = {
20
+ RRDENG_PAGE_TYPE_GORILLA_32BIT,
21
+ RRDENG_PAGE_TYPE_ARRAY_TIER1,
22
+ RRDENG_PAGE_TYPE_ARRAY_TIER1,
23
+ RRDENG_PAGE_TYPE_ARRAY_TIER1,
24
+ RRDENG_PAGE_TYPE_ARRAY_TIER1};
25
26
#if defined(ENV32BIT)
27
size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192};
@@ -24,14 +29,14 @@ size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192};
29
size_t tier_page_size[RRD_STORAGE_TIERS] = {4096, 2048, 384, 384, 384};
30
#endif
31
27
-#if PAGE_TYPE_MAX != 2
32
+#if RRDENG_PAGE_TYPE_MAX != 2
33
#error PAGE_TYPE_MAX is not 2 - you need to add allocations here
34
#endif
35
36
size_t page_type_size[256] = {
32
- [PAGE_METRICS] = sizeof(storage_number),
33
- [PAGE_TIER] = sizeof(storage_number_tier1_t),
34
- [PAGE_GORILLA_METRICS] = sizeof(storage_number)
37
+ [RRDENG_PAGE_TYPE_ARRAY_32BIT] = sizeof(storage_number),
38
+ [RRDENG_PAGE_TYPE_ARRAY_TIER1] = sizeof(storage_number_tier1_t),
39
+ [RRDENG_PAGE_TYPE_GORILLA_32BIT] = sizeof(storage_number)
40
};
41
42
__attribute__((constructor)) void initialize_multidb_ctx(void) {
@@ -457,14 +462,14 @@ static PGD *rrdeng_alloc_new_page_data(struct rrdeng_collect_handle *handle, siz
462
*data_size = size;
463
464
switch (ctx->config.page_type) {
460
- case PAGE_METRICS:
461
- case PAGE_TIER:
465
+ case RRDENG_PAGE_TYPE_ARRAY_32BIT:
466
+ case RRDENG_PAGE_TYPE_ARRAY_TIER1:
467
d = pgd_create(ctx->config.page_type, slots);
468
break;
464
- case PAGE_GORILLA_METRICS:
469
+ case RRDENG_PAGE_TYPE_GORILLA_32BIT:
470
// ignore slots, and use the fixed number of slots per gorilla buffer.
471
// gorilla will automatically add more buffers if needed.
467
- d = pgd_create(ctx->config.page_type, GORILLA_BUFFER_SLOTS);
472
+ d = pgd_create(ctx->config.page_type, RRDENG_GORILLA_32BIT_BUFFER_SLOTS);
473
break;
474
default:
475
fatal("Unknown page type: %uc\n", ctx->config.page_type);
@@ -1165,7 +1170,7 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
1170
1171
ctx->config.tier = (int)tier;
1172
ctx->config.page_type = tier_page_type[tier];
1168
- ctx->config.global_compress_alg = RRD_LZ4;
1173
+ ctx->config.global_compress_alg = RRDENG_COMPRESSION_LZ4;
1174
if (disk_space_mb < RRDENG_MIN_DISK_SPACE_MB)
1175
disk_space_mb = RRDENG_MIN_DISK_SPACE_MB;
1176
ctx->config.max_disk_space = disk_space_mb * 1048576LLU;
src/database/rrd.h
+4
-8
@@ -1002,15 +1002,11 @@ typedef enum __attribute__ ((__packed__)) {
1002
1003
// Streaming configuration
1004
RRDHOST_OPTION_SENDER_ENABLED = (1 << 2), // set when the host is configured to send metrics to a parent
1005
+ RRDHOST_OPTION_REPLICATION = (1 << 3), // when set, we support replication for this host
1006
1006
- // Configuration options
1007
- RRDHOST_OPTION_DELETE_OBSOLETE_CHARTS = (1 << 3), // delete files of obsolete charts
1008
- RRDHOST_OPTION_DELETE_ORPHAN_HOST = (1 << 4), // delete the entire host when orphan
1009
-
1010
- RRDHOST_OPTION_REPLICATION = (1 << 5), // when set, we support replication for this host
1011
-
1012
- RRDHOST_OPTION_VIRTUAL_HOST = (1 << 6), // when set, this host is a virtual one
1013
- RRDHOST_OPTION_EPHEMERAL_HOST = (1 << 7), // when set, this host is an ephemeral one
1007
+ // Other options
1008
+ RRDHOST_OPTION_VIRTUAL_HOST = (1 << 4), // when set, this host is a virtual one
1009
+ RRDHOST_OPTION_EPHEMERAL_HOST = (1 << 5), // when set, this host is an ephemeral one
1010
} RRDHOST_OPTIONS;
1011
1012
#define rrdhost_option_check(host, flag) ((host)->options & (flag))
src/database/rrdhost.c
-6
@@ -360,12 +360,6 @@ int is_legacy = 1;
360
361
rrdset_index_init(host);
362
363
- if(config_get_boolean(CONFIG_SECTION_DB, "delete obsolete charts files", 1))
364
- rrdhost_option_set(host, RRDHOST_OPTION_DELETE_OBSOLETE_CHARTS);
365
-
366
- if(config_get_boolean(CONFIG_SECTION_DB, "delete orphan hosts files", 1) && !is_localhost)
367
- rrdhost_option_set(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST);
368
-
363
char filename[FILENAME_MAX + 1];
364
if(is_localhost)
365
host->cache_dir = strdupz(netdata_configured_cache_dir);
src/libnetdata/bitmap.h
new
+81
@@ -0,0 +1,81 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_BITMAP_H
4
+#define NETDATA_BITMAP_H
5
+
6
+#ifdef ENV32BIT
7
+
8
+typedef struct bitmapX {
9
+ uint32_t bits;
10
+ uint32_t data[];
11
+} BITMAPX;
12
+
13
+typedef struct bitmap256 {
14
+ uint32_t bits;
15
+ uint32_t data[256 / 32];
16
+} BITMAP256;
17
+
18
+typedef struct bitmap1024 {
19
+ uint32_t bits;
20
+ uint32_t data[1024 / 32];
21
+} BITMAP1024;
22
+
23
+static inline BITMAPX *bitmapX_create(uint32_t bits) {
24
+ BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint32_t) * ((bits + 31) / 32));
25
+ uint32_t *p = (uint32_t *)&bmp->bits;
26
+ *p = bits;
27
+ return bmp;
28
+}
29
+
30
+#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 5] & (1U << ((idx) & 31)))
31
+#define bitmapX_set_bit(ptr, idx, value) do { \
32
+ register uint32_t _bitmask = 1U << ((idx) & 31); \
33
+ if (value) \
34
+ (ptr)->data[(idx) >> 5] |= _bitmask; \
35
+ else \
36
+ (ptr)->data[(idx) >> 5] &= ~_bitmask; \
37
+} while(0)
38
+
39
+#else // 64bit version of bitmaps
40
+
41
+typedef struct bitmapX {
42
+ uint32_t bits;
43
+ uint64_t data[];
44
+} BITMAPX;
45
+
46
+typedef struct bitmap256 {
47
+ uint32_t bits;
48
+ uint64_t data[256 / 64];
49
+} BITMAP256;
50
+
51
+typedef struct bitmap1024 {
52
+ uint32_t bits;
53
+ uint64_t data[1024 / 64];
54
+} BITMAP1024;
55
+
56
+static inline BITMAPX *bitmapX_create(uint32_t bits) {
57
+ BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint64_t) * ((bits + 63) / 64));
58
+ bmp->bits = bits;
59
+ return bmp;
60
+}
61
+
62
+#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 6] & (1ULL << ((idx) & 63)))
63
+#define bitmapX_set_bit(ptr, idx, value) do { \
64
+ register uint64_t _bitmask = 1ULL << ((idx) & 63); \
65
+ if (value) \
66
+ (ptr)->data[(idx) >> 6] |= _bitmask; \
67
+ else \
68
+ (ptr)->data[(idx) >> 6] &= ~_bitmask; \
69
+} while(0)
70
+
71
+#endif // 64bit version of bitmaps
72
+
73
+#define BITMAPX_INITIALIZER(wanted_bits) { .bits = (wanted_bits), .data = {0} }
74
+#define BITMAP256_INITIALIZER (BITMAP256)BITMAPX_INITIALIZER(256)
75
+#define BITMAP1024_INITIALIZER (BITMAP1024)BITMAPX_INITIALIZER(1024)
76
+#define bitmap256_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx)
77
+#define bitmap256_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value)
78
+#define bitmap1024_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx)
79
+#define bitmap1024_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value)
80
+
81
+#endif //NETDATA_BITMAP_H
src/libnetdata/gorilla/gorilla.cc
+2
-2
@@ -212,7 +212,7 @@ bool gorilla_writer_serialize(const gorilla_writer_t *gw, uint8_t *dst, uint32_t
212
do {
213
const gorilla_buffer_t *next_gbuf = curr_gbuf->header.next;
214
215
- size_t bytes = GORILLA_BUFFER_SIZE;
215
+ size_t bytes = RRDENG_GORILLA_32BIT_BUFFER_SIZE;
216
if (bytes > dst_size)
217
return false;
218
@@ -232,7 +232,7 @@ uint32_t gorilla_buffer_patch(gorilla_buffer_t *gbuf) {
232
233
while (curr_gbuf->header.next) {
234
uint32_t *buf = reinterpret_cast<uint32_t *>(gbuf);
235
- gbuf = reinterpret_cast<gorilla_buffer_t *>(&buf[GORILLA_BUFFER_SLOTS]);
235
+ gbuf = reinterpret_cast<gorilla_buffer_t *>(&buf[RRDENG_GORILLA_32BIT_BUFFER_SLOTS]);
236
237
assert(((uintptr_t) (gbuf) % sizeof(uintptr_t)) == 0 &&
238
"Gorilla buffer not aligned to uintptr_t");
src/libnetdata/gorilla/gorilla.h
+2
-2
@@ -67,8 +67,8 @@ uint32_t gorilla_buffer_patch(gorilla_buffer_t *buf);
67
gorilla_reader_t gorilla_reader_init(gorilla_buffer_t *buf);
68
bool gorilla_reader_read(gorilla_reader_t *gr, uint32_t *number);
69
70
-#define GORILLA_BUFFER_SLOTS 128
71
-#define GORILLA_BUFFER_SIZE (GORILLA_BUFFER_SLOTS * sizeof(uint32_t))
70
+#define RRDENG_GORILLA_32BIT_BUFFER_SLOTS 128
71
+#define RRDENG_GORILLA_32BIT_BUFFER_SIZE (RRDENG_GORILLA_32BIT_BUFFER_SLOTS * sizeof(uint32_t))
72
73
#ifdef __cplusplus
74
}
src/libnetdata/libnetdata.h
+3
-324
@@ -246,256 +246,8 @@ size_t judy_aral_structures(void);
246
247
#define GUID_LEN 36
248
249
-// ---------------------------------------------------------------------------------------------
250
-// double linked list management
251
-// inspired by https://github.com/troydhanson/uthash/blob/master/src/utlist.h
252
-
253
-#define DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next) \
254
- do { \
255
- (item)->next = (head); \
256
- \
257
- if(likely(head)) { \
258
- (item)->prev = (head)->prev; \
259
- (head)->prev = (item); \
260
- } \
261
- else \
262
- (item)->prev = (item); \
263
- \
264
- (head) = (item); \
265
- } while (0)
266
-
267
-#define DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next) \
268
- do { \
269
- \
270
- (item)->next = NULL; \
271
- \
272
- if(likely(head)) { \
273
- (item)->prev = (head)->prev; \
274
- (head)->prev->next = (item); \
275
- (head)->prev = (item); \
276
- } \
277
- else { \
278
- (item)->prev = (item); \
279
- (head) = (item); \
280
- } \
281
- \
282
- } while (0)
283
-
284
-#define DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(head, item, prev, next) \
285
- do { \
286
- fatal_assert((head) != NULL); \
287
- fatal_assert((item)->prev != NULL); \
288
- \
289
- if((item)->prev == (item)) \
290
- /* it is the only item in the list */ \
291
- (head) = NULL; \
292
- \
293
- else if((item) == (head)) { \
294
- /* it is the first item */ \
295
- fatal_assert((item)->next != NULL); \
296
- (item)->next->prev = (item)->prev; \
297
- (head) = (item)->next; \
298
- } \
299
- else { \
300
- /* it is any other item */ \
301
- (item)->prev->next = (item)->next; \
302
- \
303
- if ((item)->next) \
304
- (item)->next->prev = (item)->prev; \
305
- else \
306
- (head)->prev = (item)->prev; \
307
- } \
308
- \
309
- (item)->next = NULL; \
310
- (item)->prev = NULL; \
311
- } while (0)
312
-
313
-#define DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(head, existing, item, prev, next) \
314
- do { \
315
- if (existing) { \
316
- fatal_assert((head) != NULL); \
317
- fatal_assert((item) != NULL); \
318
- \
319
- (item)->next = (existing); \
320
- (item)->prev = (existing)->prev; \
321
- (existing)->prev = (item); \
322
- \
323
- if ((head) == (existing)) \
324
- (head) = (item); \
325
- else \
326
- (item)->prev->next = (item); \
327
- \
328
- } \
329
- else \
330
- DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next); \
331
- \
332
- } while (0)
333
-
334
-#define DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(head, existing, item, prev, next) \
335
- do { \
336
- if (existing) { \
337
- fatal_assert((head) != NULL); \
338
- fatal_assert((item) != NULL); \
339
- \
340
- (item)->next = (existing)->next; \
341
- (item)->prev = (existing); \
342
- (existing)->next = (item); \
343
- \
344
- if ((item)->next) \
345
- (item)->next->prev = (item); \
346
- else \
347
- (head)->prev = (item); \
348
- } \
349
- else \
350
- DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next); \
351
- \
352
- } while (0)
353
-
354
-#define DOUBLE_LINKED_LIST_APPEND_LIST_UNSAFE(head, head2, prev, next) \
355
- do { \
356
- if (head2) { \
357
- if (head) { \
358
- __typeof(head2) _head2_last_item = (head2)->prev; \
359
- \
360
- (head2)->prev = (head)->prev; \
361
- (head)->prev->next = (head2); \
362
- \
363
- (head)->prev = _head2_last_item; \
364
- } \
365
- else \
366
- (head) = (head2); \
367
- } \
368
- } while (0)
369
-
370
-#define DOUBLE_LINKED_LIST_FOREACH_FORWARD(head, var, prev, next) \
371
- for ((var) = (head); (var) ; (var) = (var)->next)
372
-
373
-#define DOUBLE_LINKED_LIST_FOREACH_BACKWARD(head, var, prev, next) \
374
- for ((var) = (head) ? (head)->prev : NULL ; (var) ; (var) = ((var) == (head)) ? NULL : (var)->prev)
375
-
376
-// ---------------------------------------------------------------------------------------------
377
-
378
-#include "storage_number/storage_number.h"
379
-
380
-typedef struct storage_point {
381
- NETDATA_DOUBLE min; // when count > 1, this is the minimum among them
382
- NETDATA_DOUBLE max; // when count > 1, this is the maximum among them
383
- NETDATA_DOUBLE sum; // the point sum - divided by count gives the average
384
-
385
- // end_time - start_time = point duration
386
- time_t start_time_s; // the time the point starts
387
- time_t end_time_s; // the time the point ends
388
-
389
- uint32_t count; // the number of original points aggregated
390
- uint32_t anomaly_count; // the number of original points found anomalous
391
-
392
- SN_FLAGS flags; // flags stored with the point
393
-} STORAGE_POINT;
394
-
395
-#define storage_point_unset(x) do { \
396
- (x).min = (x).max = (x).sum = NAN; \
397
- (x).count = 0; \
398
- (x).anomaly_count = 0; \
399
- (x).flags = SN_FLAG_NONE; \
400
- (x).start_time_s = 0; \
401
- (x).end_time_s = 0; \
402
- } while(0)
403
-
404
-#define storage_point_empty(x, start_s, end_s) do { \
405
- (x).min = (x).max = (x).sum = NAN; \
406
- (x).count = 1; \
407
- (x).anomaly_count = 0; \
408
- (x).flags = SN_FLAG_NONE; \
409
- (x).start_time_s = start_s; \
410
- (x).end_time_s = end_s; \
411
- } while(0)
412
-
413
-#define STORAGE_POINT_UNSET (STORAGE_POINT){ .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }
414
-
415
-#define storage_point_is_unset(x) (!(x).count)
416
-#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))
417
-#define storage_point_is_zero(x) (!(x).count || (netdata_double_is_zero((x).min) && netdata_double_is_zero((x).max) && netdata_double_is_zero((x).sum) && (x).anomaly_count == 0))
418
-
419
-#define storage_point_merge_to(dst, src) do { \
420
- if(storage_point_is_unset(dst)) \
421
- (dst) = (src); \
422
- \
423
- else if(!storage_point_is_unset(src) && \
424
- !storage_point_is_gap(src)) { \
425
- \
426
- if((src).start_time_s < (dst).start_time_s) \
427
- (dst).start_time_s = (src).start_time_s;\
428
- \
429
- if((src).end_time_s > (dst).end_time_s) \
430
- (dst).end_time_s = (src).end_time_s; \
431
- \
432
- if((src).min < (dst).min) \
433
- (dst).min = (src).min; \
434
- \
435
- if((src).max > (dst).max) \
436
- (dst).max = (src).max; \
437
- \
438
- (dst).sum += (src).sum; \
439
- \
440
- (dst).count += (src).count; \
441
- (dst).anomaly_count += (src).anomaly_count; \
442
- \
443
- (dst).flags |= (src).flags & SN_FLAG_RESET; \
444
- } \
445
-} while(0)
446
-
447
-#define storage_point_add_to(dst, src) do { \
448
- if(storage_point_is_unset(dst)) \
449
- (dst) = (src); \
450
- \
451
- else if(!storage_point_is_unset(src) && \
452
- !storage_point_is_gap(src)) { \
453
- \
454
- if((src).start_time_s < (dst).start_time_s) \
455
- (dst).start_time_s = (src).start_time_s;\
456
- \
457
- if((src).end_time_s > (dst).end_time_s) \
458
- (dst).end_time_s = (src).end_time_s; \
459
- \
460
- (dst).min += (src).min; \
461
- (dst).max += (src).max; \
462
- (dst).sum += (src).sum; \
463
- \
464
- (dst).count += (src).count; \
465
- (dst).anomaly_count += (src).anomaly_count; \
466
- \
467
- (dst).flags |= (src).flags & SN_FLAG_RESET; \
468
- } \
469
-} while(0)
470
-
471
-#define storage_point_make_positive(sp) do { \
472
- if(!storage_point_is_unset(sp) && \
473
- !storage_point_is_gap(sp)) { \
474
- \
475
- if(unlikely(signbit((sp).sum))) \
476
- (sp).sum = -(sp).sum; \
477
- \
478
- if(unlikely(signbit((sp).min))) \
479
- (sp).min = -(sp).min; \
480
- \
481
- if(unlikely(signbit((sp).max))) \
482
- (sp).max = -(sp).max; \
483
- \
484
- if(unlikely((sp).min > (sp).max)) { \
485
- NETDATA_DOUBLE t = (sp).min; \
486
- (sp).min = (sp).max; \
487
- (sp).max = t; \
488
- } \
489
- } \
490
-} while(0)
491
-
492
-#define storage_point_anomaly_rate(sp) \
493
- (NETDATA_DOUBLE)(storage_point_is_unset(sp) ? 0.0 : (NETDATA_DOUBLE)((sp).anomaly_count) * 100.0 / (NETDATA_DOUBLE)((sp).count))
494
-
495
-#define storage_point_average_value(sp) \
496
- ((sp).count ? (sp).sum / (NETDATA_DOUBLE)((sp).count) : 0.0)
497
-
498
-// ---------------------------------------------------------------------------------------------
249
+#include "linked-lists.h"
250
+#include "storage-point.h"
251
252
void netdata_fix_chart_id(char *s);
253
void netdata_fix_chart_name(char *s);
@@ -600,80 +352,7 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c
352
// Taken from linux kernel
353
#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
354
603
-#ifdef ENV32BIT
604
-
605
-typedef struct bitmapX {
606
- uint32_t bits;
607
- uint32_t data[];
608
-} BITMAPX;
609
-
610
-typedef struct bitmap256 {
611
- uint32_t bits;
612
- uint32_t data[256 / 32];
613
-} BITMAP256;
614
-
615
-typedef struct bitmap1024 {
616
- uint32_t bits;
617
- uint32_t data[1024 / 32];
618
-} BITMAP1024;
619
-
620
-static inline BITMAPX *bitmapX_create(uint32_t bits) {
621
- BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint32_t) * ((bits + 31) / 32));
622
- uint32_t *p = (uint32_t *)&bmp->bits;
623
- *p = bits;
624
- return bmp;
625
-}
626
-
627
-#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 5] & (1U << ((idx) & 31)))
628
-#define bitmapX_set_bit(ptr, idx, value) do { \
629
- register uint32_t _bitmask = 1U << ((idx) & 31); \
630
- if (value) \
631
- (ptr)->data[(idx) >> 5] |= _bitmask; \
632
- else \
633
- (ptr)->data[(idx) >> 5] &= ~_bitmask; \
634
-} while(0)
635
-
636
-#else // 64bit version of bitmaps
637
-
638
-typedef struct bitmapX {
639
- uint32_t bits;
640
- uint64_t data[];
641
-} BITMAPX;
642
-
643
-typedef struct bitmap256 {
644
- uint32_t bits;
645
- uint64_t data[256 / 64];
646
-} BITMAP256;
647
-
648
-typedef struct bitmap1024 {
649
- uint32_t bits;
650
- uint64_t data[1024 / 64];
651
-} BITMAP1024;
652
-
653
-static inline BITMAPX *bitmapX_create(uint32_t bits) {
654
- BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint64_t) * ((bits + 63) / 64));
655
- bmp->bits = bits;
656
- return bmp;
657
-}
658
-
659
-#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 6] & (1ULL << ((idx) & 63)))
660
-#define bitmapX_set_bit(ptr, idx, value) do { \
661
- register uint64_t _bitmask = 1ULL << ((idx) & 63); \
662
- if (value) \
663
- (ptr)->data[(idx) >> 6] |= _bitmask; \
664
- else \
665
- (ptr)->data[(idx) >> 6] &= ~_bitmask; \
666
-} while(0)
667
-
668
-#endif // 64bit version of bitmaps
669
-
670
-#define BITMAPX_INITIALIZER(wanted_bits) { .bits = (wanted_bits), .data = {0} }
671
-#define BITMAP256_INITIALIZER (BITMAP256)BITMAPX_INITIALIZER(256)
672
-#define BITMAP1024_INITIALIZER (BITMAP1024)BITMAPX_INITIALIZER(1024)
673
-#define bitmap256_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx)
674
-#define bitmap256_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value)
675
-#define bitmap1024_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx)
676
-#define bitmap1024_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value)
355
+#include "bitmap.h"
356
357
#define COMPRESSION_MAX_CHUNK 0x4000
358
#define COMPRESSION_MAX_OVERHEAD 128
src/libnetdata/linked-lists.h
new
+133
@@ -0,0 +1,133 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_LINKED_LISTS_H
4
+#define NETDATA_LINKED_LISTS_H
5
+
6
+// ---------------------------------------------------------------------------------------------
7
+// double linked list management
8
+// inspired by https://github.com/troydhanson/uthash/blob/master/src/utlist.h
9
+
10
+#define DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next) \
11
+ do { \
12
+ (item)->next = (head); \
13
+ \
14
+ if(likely(head)) { \
15
+ (item)->prev = (head)->prev; \
16
+ (head)->prev = (item); \
17
+ } \
18
+ else \
19
+ (item)->prev = (item); \
20
+ \
21
+ (head) = (item); \
22
+ } while (0)
23
+
24
+#define DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next) \
25
+ do { \
26
+ \
27
+ (item)->next = NULL; \
28
+ \
29
+ if(likely(head)) { \
30
+ (item)->prev = (head)->prev; \
31
+ (head)->prev->next = (item); \
32
+ (head)->prev = (item); \
33
+ } \
34
+ else { \
35
+ (item)->prev = (item); \
36
+ (head) = (item); \
37
+ } \
38
+ \
39
+ } while (0)
40
+
41
+#define DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(head, item, prev, next) \
42
+ do { \
43
+ fatal_assert((head) != NULL); \
44
+ fatal_assert((item)->prev != NULL); \
45
+ \
46
+ if((item)->prev == (item)) \
47
+ /* it is the only item in the list */ \
48
+ (head) = NULL; \
49
+ \
50
+ else if((item) == (head)) { \
51
+ /* it is the first item */ \
52
+ fatal_assert((item)->next != NULL); \
53
+ (item)->next->prev = (item)->prev; \
54
+ (head) = (item)->next; \
55
+ } \
56
+ else { \
57
+ /* it is any other item */ \
58
+ (item)->prev->next = (item)->next; \
59
+ \
60
+ if ((item)->next) \
61
+ (item)->next->prev = (item)->prev; \
62
+ else \
63
+ (head)->prev = (item)->prev; \
64
+ } \
65
+ \
66
+ (item)->next = NULL; \
67
+ (item)->prev = NULL; \
68
+ } while (0)
69
+
70
+#define DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(head, existing, item, prev, next) \
71
+ do { \
72
+ if (existing) { \
73
+ fatal_assert((head) != NULL); \
74
+ fatal_assert((item) != NULL); \
75
+ \
76
+ (item)->next = (existing); \
77
+ (item)->prev = (existing)->prev; \
78
+ (existing)->prev = (item); \
79
+ \
80
+ if ((head) == (existing)) \
81
+ (head) = (item); \
82
+ else \
83
+ (item)->prev->next = (item); \
84
+ \
85
+ } \
86
+ else \
87
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next); \
88
+ \
89
+ } while (0)
90
+
91
+#define DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(head, existing, item, prev, next) \
92
+ do { \
93
+ if (existing) { \
94
+ fatal_assert((head) != NULL); \
95
+ fatal_assert((item) != NULL); \
96
+ \
97
+ (item)->next = (existing)->next; \
98
+ (item)->prev = (existing); \
99
+ (existing)->next = (item); \
100
+ \
101
+ if ((item)->next) \
102
+ (item)->next->prev = (item); \
103
+ else \
104
+ (head)->prev = (item); \
105
+ } \
106
+ else \
107
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next); \
108
+ \
109
+ } while (0)
110
+
111
+#define DOUBLE_LINKED_LIST_APPEND_LIST_UNSAFE(head, head2, prev, next) \
112
+ do { \
113
+ if (head2) { \
114
+ if (head) { \
115
+ __typeof(head2) _head2_last_item = (head2)->prev; \
116
+ \
117
+ (head2)->prev = (head)->prev; \
118
+ (head)->prev->next = (head2); \
119
+ \
120
+ (head)->prev = _head2_last_item; \
121
+ } \
122
+ else \
123
+ (head) = (head2); \
124
+ } \
125
+ } while (0)
126
+
127
+#define DOUBLE_LINKED_LIST_FOREACH_FORWARD(head, var, prev, next) \
128
+ for ((var) = (head); (var) ; (var) = (var)->next)
129
+
130
+#define DOUBLE_LINKED_LIST_FOREACH_BACKWARD(head, var, prev, next) \
131
+ for ((var) = (head) ? (head)->prev : NULL ; (var) ; (var) = ((var) == (head)) ? NULL : (var)->prev)
132
+
133
+#endif //NETDATA_LINKED_LISTS_H
src/libnetdata/storage-point.h
new
+127
@@ -0,0 +1,127 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_STORAGE_POINT_H
4
+#define NETDATA_STORAGE_POINT_H
5
+
6
+#include "storage_number/storage_number.h"
7
+
8
+typedef struct storage_point {
9
+ NETDATA_DOUBLE min; // when count > 1, this is the minimum among them
10
+ NETDATA_DOUBLE max; // when count > 1, this is the maximum among them
11
+ NETDATA_DOUBLE sum; // the point sum - divided by count gives the average
12
+
13
+ // end_time - start_time = point duration
14
+ time_t start_time_s; // the time the point starts
15
+ time_t end_time_s; // the time the point ends
16
+
17
+ uint32_t count; // the number of original points aggregated
18
+ uint32_t anomaly_count; // the number of original points found anomalous
19
+
20
+ SN_FLAGS flags; // flags stored with the point
21
+} STORAGE_POINT;
22
+
23
+#define storage_point_unset(x) do { \
24
+ (x).min = (x).max = (x).sum = NAN; \
25
+ (x).count = 0; \
26
+ (x).anomaly_count = 0; \
27
+ (x).flags = SN_FLAG_NONE; \
28
+ (x).start_time_s = 0; \
29
+ (x).end_time_s = 0; \
30
+ } while(0)
31
+
32
+#define storage_point_empty(x, start_s, end_s) do { \
33
+ (x).min = (x).max = (x).sum = NAN; \
34
+ (x).count = 1; \
35
+ (x).anomaly_count = 0; \
36
+ (x).flags = SN_FLAG_NONE; \
37
+ (x).start_time_s = start_s; \
38
+ (x).end_time_s = end_s; \
39
+ } while(0)
40
+
41
+#define STORAGE_POINT_UNSET (STORAGE_POINT){ .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }
42
+
43
+#define storage_point_is_unset(x) (!(x).count)
44
+#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))
45
+#define storage_point_is_zero(x) (!(x).count || (netdata_double_is_zero((x).min) && netdata_double_is_zero((x).max) && netdata_double_is_zero((x).sum) && (x).anomaly_count == 0))
46
+
47
+#define storage_point_merge_to(dst, src) do { \
48
+ if(storage_point_is_unset(dst)) \
49
+ (dst) = (src); \
50
+ \
51
+ else if(!storage_point_is_unset(src) && \
52
+ !storage_point_is_gap(src)) { \
53
+ \
54
+ if((src).start_time_s < (dst).start_time_s) \
55
+ (dst).start_time_s = (src).start_time_s;\
56
+ \
57
+ if((src).end_time_s > (dst).end_time_s) \
58
+ (dst).end_time_s = (src).end_time_s; \
59
+ \
60
+ if((src).min < (dst).min) \
61
+ (dst).min = (src).min; \
62
+ \
63
+ if((src).max > (dst).max) \
64
+ (dst).max = (src).max; \
65
+ \
66
+ (dst).sum += (src).sum; \
67
+ \
68
+ (dst).count += (src).count; \
69
+ (dst).anomaly_count += (src).anomaly_count; \
70
+ \
71
+ (dst).flags |= (src).flags & SN_FLAG_RESET; \
72
+ } \
73
+} while(0)
74
+
75
+#define storage_point_add_to(dst, src) do { \
76
+ if(storage_point_is_unset(dst)) \
77
+ (dst) = (src); \
78
+ \
79
+ else if(!storage_point_is_unset(src) && \
80
+ !storage_point_is_gap(src)) { \
81
+ \
82
+ if((src).start_time_s < (dst).start_time_s) \
83
+ (dst).start_time_s = (src).start_time_s;\
84
+ \
85
+ if((src).end_time_s > (dst).end_time_s) \
86
+ (dst).end_time_s = (src).end_time_s; \
87
+ \
88
+ (dst).min += (src).min; \
89
+ (dst).max += (src).max; \
90
+ (dst).sum += (src).sum; \
91
+ \
92
+ (dst).count += (src).count; \
93
+ (dst).anomaly_count += (src).anomaly_count; \
94
+ \
95
+ (dst).flags |= (src).flags & SN_FLAG_RESET; \
96
+ } \
97
+} while(0)
98
+
99
+#define storage_point_make_positive(sp) do { \
100
+ if(!storage_point_is_unset(sp) && \
101
+ !storage_point_is_gap(sp)) { \
102
+ \
103
+ if(unlikely(signbit((sp).sum))) \
104
+ (sp).sum = -(sp).sum; \
105
+ \
106
+ if(unlikely(signbit((sp).min))) \
107
+ (sp).min = -(sp).min; \
108
+ \
109
+ if(unlikely(signbit((sp).max))) \
110
+ (sp).max = -(sp).max; \
111
+ \
112
+ if(unlikely((sp).min > (sp).max)) { \
113
+ NETDATA_DOUBLE t = (sp).min; \
114
+ (sp).min = (sp).max; \
115
+ (sp).max = t; \
116
+ } \
117
+ } \
118
+} while(0)
119
+
120
+#define storage_point_anomaly_rate(sp) \
121
+ (NETDATA_DOUBLE)(storage_point_is_unset(sp) ? 0.0 : (NETDATA_DOUBLE)((sp).anomaly_count) * 100.0 / (NETDATA_DOUBLE)((sp).count))
122
+
123
+#define storage_point_average_value(sp) \
124
+ ((sp).count ? (sp).sum / (NETDATA_DOUBLE)((sp).count) : 0.0)
125
+
126
+
127
+#endif //NETDATA_STORAGE_POINT_H