DBENGINE v2 - improvements part 11 (#14337)
* acquiring / releasing interface for metrics * metrics registry statistics * cleanup metrics registry by deleting metrics when they dont have retention anymore; do not double copy the data of pages to be flushed * print the tier in retention summary * Open files with buffered instead of direct I/O (test) * added more metrics stats and fixed unittest * rename writer functions to avoid confusion with refcounting * do not release a metric that is not acquired * Revert to use direct I/O on write -- use direct I/O on read as well * keep track of ARAL overhead and add it to the memory chart * aral full check via api * Cleanup * give names to ARALs and PGCs * aral improvements * restore query expansion to the future * prefer higher resolution tier when switching plans * added extent read statistics * smoother joining of tiers at query engine * fine tune aral max allocation size * aral restructuring to hide its internals from the rest of netdata * aral restructuring; addtion of defrag option to aral to keep the linked list sorted - enabled by default to test it * fully async aral * some statistics and cleanup * fix infinite loop while calculating retention * aral docs and defragmenting disabled by default * fix bug and add optimization when defragmenter is not enabled * aral stress test * aral speed report and documentation * added internal checks that all pages are full * improve internal log about metrics deletion * metrics registry uses one aral per partition * metrics registry aral max size to 512 elements per page * remove data_structures/README.md dependency --------- Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Costa Tsaousis committed
Jan 30, 2023 at 20:36 UTC
7f8f11eb373dfc7bf6ac5a03e57a1b03487a279e
40 files changed
+1915
-818
CMakeLists.txt
+2
-2
@@ -453,8 +453,8 @@ set(LIBNETDATA_FILES
453
libnetdata/adaptive_resortable_list/adaptive_resortable_list.h
454
libnetdata/config/appconfig.c
455
libnetdata/config/appconfig.h
456
- libnetdata/arrayalloc/arrayalloc.c
457
- libnetdata/arrayalloc/arrayalloc.h
456
+ libnetdata/aral/aral.c
457
+ libnetdata/aral/aral.h
458
libnetdata/avl/avl.c
459
libnetdata/avl/avl.h
460
libnetdata/buffer/buffer.c
Makefile.am
+2
-2
@@ -131,8 +131,8 @@ LIBNETDATA_FILES = \
131
libnetdata/adaptive_resortable_list/adaptive_resortable_list.h \
132
libnetdata/config/appconfig.c \
133
libnetdata/config/appconfig.h \
134
- libnetdata/arrayalloc/arrayalloc.c \
135
- libnetdata/arrayalloc/arrayalloc.h \
134
+ libnetdata/aral/aral.c \
135
+ libnetdata/aral/aral.h \
136
libnetdata/avl/avl.c \
137
libnetdata/avl/avl.h \
138
libnetdata/buffer/buffer.c \
collectors/apps.plugin/apps_plugin.c
+2
-2
@@ -975,7 +975,7 @@ static inline struct pid_stat *get_pid_entry(pid_t pid) {
975
init_pid_fds(p, 0, p->fds_size);
976
p->pid = pid;
977
978
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(root_of_pids, p, prev, next);
978
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(root_of_pids, p, prev, next);
979
980
all_pids[pid] = p;
981
all_pids_count++;
@@ -993,7 +993,7 @@ static inline void del_pid_entry(pid_t pid) {
993
994
debug_log("process %d %s exited, deleting it.", pid, p->comm);
995
996
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(root_of_pids, p, prev, next);
996
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(root_of_pids, p, prev, next);
997
998
// free the filename
999
#ifndef __FreeBSD__
configure.ac
+1
-1
@@ -1730,7 +1730,7 @@ AC_CONFIG_FILES([
1730
libnetdata/Makefile
1731
libnetdata/tests/Makefile
1732
libnetdata/adaptive_resortable_list/Makefile
1733
- libnetdata/arrayalloc/Makefile
1733
+ libnetdata/aral/Makefile
1734
libnetdata/avl/Makefile
1735
libnetdata/buffer/Makefile
1736
libnetdata/clocks/Makefile
daemon/global_statistics.c
+116
-2
@@ -243,6 +243,9 @@ static void global_statistics_charts(void) {
243
global_statistics_copy(&gs, GLOBAL_STATS_RESET_WEB_USEC_MAX);
244
getrusage(RUSAGE_SELF, &me);
245
246
+ size_t aral_structures, aral_malloc_allocated, aral_malloc_used, aral_mmap_allocated, aral_mmap_used;
247
+ aral_get_size_statistics(&aral_structures, &aral_malloc_allocated, &aral_malloc_used, &aral_mmap_allocated, &aral_mmap_used);
248
+
249
// ----------------------------------------------------------------
250
251
{
@@ -292,6 +295,7 @@ static void global_statistics_charts(void) {
295
static RRDDIM *rd_replication = NULL;
296
static RRDDIM *rd_buffers = NULL;
297
static RRDDIM *rd_workers = NULL;
298
+ static RRDDIM *rd_aral = NULL;
299
static RRDDIM *rd_other = NULL;
300
301
if (unlikely(!st_memory)) {
@@ -322,6 +326,7 @@ static void global_statistics_charts(void) {
326
rd_replication = rrddim_add(st_memory, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
327
rd_buffers = rrddim_add(st_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
328
rd_workers = rrddim_add(st_memory, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
329
+ rd_aral = rrddim_add(st_memory, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
330
rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
331
}
332
@@ -355,6 +360,7 @@ static void global_statistics_charts(void) {
360
rrddim_set_by_pointer(st_memory, rd_replication, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + (collected_number)replication_allocated_memory());
361
rrddim_set_by_pointer(st_memory, rd_buffers, (collected_number)buffers);
362
rrddim_set_by_pointer(st_memory, rd_workers, (collected_number) workers_allocated_memory());
363
+ rrddim_set_by_pointer(st_memory, rd_aral, (collected_number) aral_structures);
364
rrddim_set_by_pointer(st_memory, rd_other, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other));
365
366
rrdset_done(st_memory);
@@ -374,6 +380,7 @@ static void global_statistics_charts(void) {
380
static RRDDIM *rd_cbuffers_streaming = NULL;
381
static RRDDIM *rd_buffers_replication = NULL;
382
static RRDDIM *rd_buffers_web = NULL;
383
+ static RRDDIM *rd_buffers_aral = NULL;
384
385
if (unlikely(!st_memory_buffers)) {
386
st_memory_buffers = rrdset_create_localhost(
@@ -402,6 +409,7 @@ static void global_statistics_charts(void) {
409
rd_cbuffers_streaming = rrddim_add(st_memory_buffers, "streaming cbuf", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
410
rd_buffers_replication = rrddim_add(st_memory_buffers, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
411
rd_buffers_web = rrddim_add(st_memory_buffers, "web", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
412
+ rd_buffers_aral = rrddim_add(st_memory_buffers, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
413
}
414
415
rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory());
@@ -416,6 +424,7 @@ static void global_statistics_charts(void) {
424
rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming);
425
rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers());
426
rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web);
427
+ rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aral, (collected_number)(aral_malloc_allocated + aral_mmap_allocated) - (collected_number)(aral_malloc_used + aral_mmap_used));
428
429
rrdset_done(st_memory_buffers);
430
}
@@ -1885,6 +1894,111 @@ static void dbengine2_statistics_charts(void) {
1894
}
1895
#endif
1896
1897
+ {
1898
+ static RRDSET *st_mrg_metrics = NULL;
1899
+ static RRDDIM *rd_mrg_metrics = NULL;
1900
+ static RRDDIM *rd_mrg_acquired = NULL;
1901
+ static RRDDIM *rd_mrg_collected = NULL;
1902
+ static RRDDIM *rd_mrg_with_retention = NULL;
1903
+ static RRDDIM *rd_mrg_without_retention = NULL;
1904
+ static RRDDIM *rd_mrg_multiple_writers = NULL;
1905
+
1906
+ if (unlikely(!st_mrg_metrics)) {
1907
+ st_mrg_metrics = rrdset_create_localhost(
1908
+ "netdata",
1909
+ "dbengine_metrics",
1910
+ NULL,
1911
+ "dbengine metrics",
1912
+ NULL,
1913
+ "Netdata Metrics in Metrics Registry",
1914
+ "metrics",
1915
+ "netdata",
1916
+ "stats",
1917
+ priority,
1918
+ localhost->rrd_update_every,
1919
+ RRDSET_TYPE_LINE);
1920
+
1921
+ rd_mrg_metrics = rrddim_add(st_mrg_metrics, "all", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1922
+ rd_mrg_acquired = rrddim_add(st_mrg_metrics, "acquired", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1923
+ rd_mrg_collected = rrddim_add(st_mrg_metrics, "collected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1924
+ rd_mrg_with_retention = rrddim_add(st_mrg_metrics, "with retention", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1925
+ rd_mrg_without_retention = rrddim_add(st_mrg_metrics, "without retention", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1926
+ rd_mrg_multiple_writers = rrddim_add(st_mrg_metrics, "multi-collected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1927
+ }
1928
+ priority++;
1929
+
1930
+ rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_metrics, (collected_number)mrg_stats.entries);
1931
+ rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_acquired, (collected_number)mrg_stats.entries_referenced);
1932
+ rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_collected, (collected_number)mrg_stats.writers);
1933
+ rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_with_retention, (collected_number)mrg_stats.entries_with_retention);
1934
+ rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_without_retention, (collected_number)mrg_stats.entries - (collected_number)mrg_stats.entries_with_retention);
1935
+ rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_multiple_writers, (collected_number)mrg_stats.writers_conflicts);
1936
+
1937
+ rrdset_done(st_mrg_metrics);
1938
+ }
1939
+
1940
+ {
1941
+ static RRDSET *st_mrg_ops = NULL;
1942
+ static RRDDIM *rd_mrg_add = NULL;
1943
+ static RRDDIM *rd_mrg_del = NULL;
1944
+ static RRDDIM *rd_mrg_search = NULL;
1945
+
1946
+ if (unlikely(!st_mrg_ops)) {
1947
+ st_mrg_ops = rrdset_create_localhost(
1948
+ "netdata",
1949
+ "dbengine_metrics_registry_operations",
1950
+ NULL,
1951
+ "dbengine metrics",
1952
+ NULL,
1953
+ "Netdata Metrics Registry Operations",
1954
+ "metrics",
1955
+ "netdata",
1956
+ "stats",
1957
+ priority,
1958
+ localhost->rrd_update_every,
1959
+ RRDSET_TYPE_LINE);
1960
+
1961
+ rd_mrg_add = rrddim_add(st_mrg_ops, "add", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1962
+ rd_mrg_del = rrddim_add(st_mrg_ops, "delete", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1963
+ rd_mrg_search = rrddim_add(st_mrg_ops, "search", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1964
+ }
1965
+ priority++;
1966
+
1967
+ rrddim_set_by_pointer(st_mrg_ops, rd_mrg_add, (collected_number)mrg_stats.additions);
1968
+ rrddim_set_by_pointer(st_mrg_ops, rd_mrg_del, (collected_number)mrg_stats.deletions);
1969
+ rrddim_set_by_pointer(st_mrg_ops, rd_mrg_search, (collected_number)mrg_stats.search_hits + (collected_number)mrg_stats.search_misses);
1970
+
1971
+ rrdset_done(st_mrg_ops);
1972
+ }
1973
+
1974
+ {
1975
+ static RRDSET *st_mrg_references = NULL;
1976
+ static RRDDIM *rd_mrg_references = NULL;
1977
+
1978
+ if (unlikely(!st_mrg_references)) {
1979
+ st_mrg_references = rrdset_create_localhost(
1980
+ "netdata",
1981
+ "dbengine_metrics_registry_references",
1982
+ NULL,
1983
+ "dbengine metrics",
1984
+ NULL,
1985
+ "Netdata Metrics Registry References",
1986
+ "references",
1987
+ "netdata",
1988
+ "stats",
1989
+ priority,
1990
+ localhost->rrd_update_every,
1991
+ RRDSET_TYPE_LINE);
1992
+
1993
+ rd_mrg_references = rrddim_add(st_mrg_references, "references", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1994
+ }
1995
+ priority++;
1996
+
1997
+ rrddim_set_by_pointer(st_mrg_references, rd_mrg_references, (collected_number)mrg_stats.current_references);
1998
+
1999
+ rrdset_done(st_mrg_references);
2000
+ }
2001
+
2002
{
2003
static RRDSET *st_cache_hit_ratio = NULL;
2004
static RRDDIM *rd_hit_ratio = NULL;
@@ -3817,7 +3931,7 @@ static void workers_threads_cleanup(struct worker_utilization *wu) {
3931
3932
if(!t->enabled) {
3933
JudyLDel(&workers_by_pid_JudyL_array, t->pid, PJE0);
3820
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(wu->threads, t, prev, next);
3934
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wu->threads, t, prev, next);
3935
freez(t);
3936
}
3937
t = next;
@@ -3844,7 +3958,7 @@ static struct worker_thread *worker_thread_create(struct worker_utilization *wu,
3958
*PValue = wt;
3959
3960
// link it
3847
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(wu->threads, wt, prev, next);
3961
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wu->threads, wt, prev, next);
3962
3963
return wt;
3964
}
database/engine/cache.c
+53
-32
@@ -16,7 +16,7 @@
16
typedef int32_t REFCOUNT;
17
#define REFCOUNT_DELETING (-100)
18
19
-// to use arrayalloc uncomment the following line:
19
+// to use ARAL uncomment the following line:
20
#define PGC_WITH_ARAL 1
21
22
typedef enum __attribute__ ((__packed__)) {
@@ -82,6 +82,8 @@ struct pgc_linked_list {
82
83
struct pgc {
84
struct {
85
+ char name[PGC_NAME_MAX + 1];
86
+
87
size_t partitions;
88
size_t clean_size;
89
size_t max_dirty_pages_per_call;
@@ -415,13 +417,25 @@ struct section_pages {
417
PGC_PAGE *base;
418
};
419
418
-static ARAL section_pages_aral = {
419
- .filename = NULL,
420
- .cache_dir = NULL,
421
- .use_mmap = false,
422
- .initial_elements = 16384 / sizeof(struct section_pages),
423
- .requested_element_size = sizeof(struct section_pages),
424
-};
420
+static ARAL *pgc_section_pages_aral = NULL;
421
+static void pgc_section_pages_static_aral_init(void) {
422
+ static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
423
+
424
+ if(unlikely(!pgc_section_pages_aral)) {
425
+ netdata_spinlock_lock(&spinlock);
426
+
427
+ // we have to check again
428
+ if(!pgc_section_pages_aral)
429
+ pgc_section_pages_aral = aral_create(
430
+ "pgc_section",
431
+ sizeof(struct section_pages),
432
+ 0,
433
+ 4096,
434
+ NULL, NULL, false, false);
435
+
436
+ netdata_spinlock_unlock(&spinlock);
437
+ }
438
+}
439
440
static inline void pgc_stats_ll_judy_change(PGC *cache, struct pgc_linked_list *ll, size_t mem_before_judyl, size_t mem_after_judyl) {
441
if(mem_after_judyl > mem_before_judyl) {
@@ -462,7 +476,7 @@ static void pgc_ll_add(PGC *cache __maybe_unused, struct pgc_linked_list *ll, PG
476
struct section_pages *sp = *section_pages_pptr;
477
if(!sp) {
478
// sp = callocz(1, sizeof(struct section_pages));
465
- sp = arrayalloc_mallocz(§ion_pages_aral);
479
+ sp = aral_mallocz(pgc_section_pages_aral);
480
memset(sp, 0, sizeof(struct section_pages));
481
482
*section_pages_pptr = sp;
@@ -473,7 +487,7 @@ static void pgc_ll_add(PGC *cache __maybe_unused, struct pgc_linked_list *ll, PG
487
488
sp->entries++;
489
sp->size += page->assumed_size;
476
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(sp->base, page, link.prev, link.next);
490
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(sp->base, page, link.prev, link.next);
491
492
if((sp->entries % cache->config.max_dirty_pages_per_call) == 0)
493
ll->version++;
@@ -484,11 +498,11 @@ static void pgc_ll_add(PGC *cache __maybe_unused, struct pgc_linked_list *ll, PG
498
// - DIRTY pages made CLEAN, depending on their accesses may be appended (accesses > 0) or prepended (accesses = 0).
499
500
if(page->accesses || page_flag_check(page, PGC_PAGE_HAS_BEEN_ACCESSED | PGC_PAGE_HAS_NO_DATA_IGNORE_ACCESSES) == PGC_PAGE_HAS_BEEN_ACCESSED) {
487
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(ll->base, page, link.prev, link.next);
501
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ll->base, page, link.prev, link.next);
502
page_flag_clear(page, PGC_PAGE_HAS_BEEN_ACCESSED);
503
}
504
else
491
- DOUBLE_LINKED_LIST_PREPEND_UNSAFE(ll->base, page, link.prev, link.next);
505
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ll->base, page, link.prev, link.next);
506
507
ll->version++;
508
}
@@ -530,7 +544,7 @@ static void pgc_ll_del(PGC *cache __maybe_unused, struct pgc_linked_list *ll, PG
544
struct section_pages *sp = *section_pages_pptr;
545
sp->entries--;
546
sp->size -= page->assumed_size;
533
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(sp->base, page, link.prev, link.next);
547
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(sp->base, page, link.prev, link.next);
548
549
if(!sp->base) {
550
size_t mem_before_judyl, mem_after_judyl;
@@ -543,13 +557,13 @@ static void pgc_ll_del(PGC *cache __maybe_unused, struct pgc_linked_list *ll, PG
557
fatal("DBENGINE CACHE: cannot delete section from Judy LL");
558
559
// freez(sp);
546
- arrayalloc_freez(§ion_pages_aral, sp);
560
+ aral_freez(pgc_section_pages_aral, sp);
561
mem_after_judyl -= sizeof(struct section_pages);
562
pgc_stats_ll_judy_change(cache, ll, mem_before_judyl, mem_after_judyl);
563
}
564
}
565
else {
552
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(ll->base, page, link.prev, link.next);
566
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ll->base, page, link.prev, link.next);
567
ll->version++;
568
}
569
@@ -565,8 +579,8 @@ static inline void page_has_been_accessed(PGC *cache, PGC_PAGE *page) {
579
580
if (flags & PGC_PAGE_CLEAN) {
581
if(pgc_ll_trylock(cache, &cache->clean)) {
568
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(cache->clean.base, page, link.prev, link.next);
569
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(cache->clean.base, page, link.prev, link.next);
582
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(cache->clean.base, page, link.prev, link.next);
583
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(cache->clean.base, page, link.prev, link.next);
584
pgc_ll_unlock(cache, &cache->clean);
585
page_flag_clear(page, PGC_PAGE_HAS_BEEN_ACCESSED);
586
}
@@ -860,7 +874,7 @@ static inline void free_this_page(PGC *cache, PGC_PAGE *page) {
874
875
// free our memory
876
#ifdef PGC_WITH_ARAL
863
- arrayalloc_freez(cache->aral, page);
877
+ aral_freez(cache->aral, page);
878
#else
879
freez(page);
880
#endif
@@ -1038,8 +1052,8 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic
1052
break;
1053
1054
if(unlikely(page_flag_check(page, PGC_PAGE_HAS_BEEN_ACCESSED | PGC_PAGE_HAS_NO_DATA_IGNORE_ACCESSES) == PGC_PAGE_HAS_BEEN_ACCESSED)) {
1041
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(cache->clean.base, page, link.prev, link.next);
1042
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(cache->clean.base, page, link.prev, link.next);
1055
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(cache->clean.base, page, link.prev, link.next);
1056
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(cache->clean.base, page, link.prev, link.next);
1057
page_flag_clear(page, PGC_PAGE_HAS_BEEN_ACCESSED);
1058
continue;
1059
}
@@ -1056,7 +1070,7 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic
1070
__atomic_add_fetch(&cache->stats.evicting_entries, 1, __ATOMIC_RELAXED);
1071
__atomic_add_fetch(&cache->stats.evicting_size, page->assumed_size, __ATOMIC_RELAXED);
1072
1059
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(pages_to_evict, page, link.prev, link.next);
1073
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(pages_to_evict, page, link.prev, link.next);
1074
1075
pages_to_evict_size += page->assumed_size;
1076
@@ -1073,8 +1087,8 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic
1087
if(!first_page_we_relocated)
1088
first_page_we_relocated = page;
1089
1076
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(cache->clean.base, page, link.prev, link.next);
1077
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(cache->clean.base, page, link.prev, link.next);
1090
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(cache->clean.base, page, link.prev, link.next);
1091
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(cache->clean.base, page, link.prev, link.next);
1092
1093
// check if we have to stop
1094
if(unlikely(++total_pages_skipped >= max_skip && !all_of_them)) {
@@ -1099,8 +1113,8 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic
1113
next = page->link.next;
1114
1115
size_t partition = pgc_indexing_partition(cache, page->metric_id);
1102
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(pages_to_evict, page, link.prev, link.next);
1103
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(pages_per_partition[partition], page, link.prev, link.next);
1116
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(pages_to_evict, page, link.prev, link.next);
1117
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(pages_per_partition[partition], page, link.prev, link.next);
1118
}
1119
1120
// remove them from the index
@@ -1178,7 +1192,7 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {
1192
__atomic_add_fetch(&cache->stats.workers_add, 1, __ATOMIC_RELAXED);
1193
1194
#ifdef PGC_WITH_ARAL
1181
- PGC_PAGE *allocation = arrayalloc_mallocz(cache->aral);
1195
+ PGC_PAGE *allocation = aral_mallocz(cache->aral);
1196
#endif
1197
PGC_PAGE *page;
1198
size_t spins = 0;
@@ -1285,7 +1299,7 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {
1299
1300
#ifdef PGC_WITH_ARAL
1301
if(allocation)
1288
- arrayalloc_freez(cache->aral, allocation);
1302
+ aral_freez(cache->aral, allocation);
1303
#endif
1304
1305
__atomic_sub_fetch(&cache->stats.workers_add, 1, __ATOMIC_RELAXED);
@@ -1713,7 +1727,8 @@ void free_all_unreferenced_clean_pages(PGC *cache) {
1727
// ----------------------------------------------------------------------------
1728
// public API
1729
1716
-PGC *pgc_create(size_t clean_size_bytes, free_clean_page_callback pgc_free_cb,
1730
+PGC *pgc_create(const char *name,
1731
+ size_t clean_size_bytes, free_clean_page_callback pgc_free_cb,
1732
size_t max_dirty_pages_per_flush,
1733
save_dirty_init_callback pgc_save_init_cb,
1734
save_dirty_page_callback pgc_save_dirty_cb,
@@ -1732,6 +1747,7 @@ PGC *pgc_create(size_t clean_size_bytes, free_clean_page_callback pgc_free_cb,
1747
max_flushes_inline = 2;
1748
1749
PGC *cache = callocz(1, sizeof(PGC));
1750
+ strncpyz(cache->config.name, name, PGC_NAME_MAX);
1751
cache->config.options = options;
1752
cache->config.clean_size = (clean_size_bytes < 1 * 1024 * 1024) ? 1 * 1024 * 1024 : clean_size_bytes;
1753
cache->config.pgc_free_clean_cb = pgc_free_cb;
@@ -1772,10 +1788,14 @@ PGC *pgc_create(size_t clean_size_bytes, free_clean_page_callback pgc_free_cb,
1788
cache->clean.stats = &cache->stats.queues.clean;
1789
1790
#ifdef PGC_WITH_ARAL
1775
- cache->aral = arrayalloc_create(sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page, 65536 / sizeof(PGC_PAGE),
1776
- NULL, NULL, false, false);
1791
+ cache->aral = aral_create(name,
1792
+ sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page,
1793
+ 0,
1794
+ 4096,
1795
+ NULL, NULL, false, false);
1796
#endif
1797
1798
+ pgc_section_pages_static_aral_init();
1799
pointer_index_init(cache);
1800
1801
return cache;
@@ -1803,7 +1823,7 @@ void pgc_destroy(PGC *cache) {
1823
else {
1824
pointer_destroy_index(cache);
1825
#ifdef PGC_WITH_ARAL
1806
- arrayalloc_destroy(cache->aral);
1826
+ aral_destroy(cache->aral);
1827
#endif
1828
freez(cache);
1829
}
@@ -2602,7 +2622,8 @@ void unittest_stress_test(void) {
2622
#endif
2623
2624
int pgc_unittest(void) {
2605
- PGC *cache = pgc_create(32 * 1024 * 1024, unittest_free_clean_page_callback,
2625
+ PGC *cache = pgc_create("test",
2626
+ 32 * 1024 * 1024, unittest_free_clean_page_callback,
2627
64, NULL, unittest_save_dirty_page_callback,
2628
10, 10, 1000, 10,
2629
PGC_OPTIONS_DEFAULT, 1, 11);
database/engine/cache.h
+3
-1
@@ -8,6 +8,7 @@
8
9
typedef struct pgc PGC;
10
typedef struct pgc_page PGC_PAGE;
11
+#define PGC_NAME_MAX 23
12
13
typedef enum __attribute__ ((__packed__)) {
14
PGC_OPTIONS_NONE = 0,
@@ -165,7 +166,8 @@ typedef void (*free_clean_page_callback)(PGC *cache, PGC_ENTRY entry);
166
typedef void (*save_dirty_page_callback)(PGC *cache, PGC_ENTRY *entries_array, PGC_PAGE **pages_array, size_t entries);
167
typedef void (*save_dirty_init_callback)(PGC *cache, Word_t section);
168
// create a cache
168
-PGC *pgc_create(size_t clean_size_bytes, free_clean_page_callback pgc_free_clean_cb,
169
+PGC *pgc_create(const char *name,
170
+ size_t clean_size_bytes, free_clean_page_callback pgc_free_clean_cb,
171
size_t max_dirty_pages_per_flush, save_dirty_init_callback pgc_save_init_cb, save_dirty_page_callback pgc_save_dirty_cb,
172
size_t max_pages_per_inline_eviction, size_t max_inline_evictors,
173
size_t max_skip_pages_per_inline_eviction,
database/engine/datafile.c
+2
-2
@@ -4,13 +4,13 @@
4
void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile)
5
{
6
uv_rwlock_wrlock(&ctx->datafiles.rwlock);
7
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(ctx->datafiles.first, datafile, prev, next);
7
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ctx->datafiles.first, datafile, prev, next);
8
uv_rwlock_wrunlock(&ctx->datafiles.rwlock);
9
}
10
11
void datafile_list_delete_unsafe(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile)
12
{
13
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(ctx->datafiles.first, datafile, prev, next);
13
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ctx->datafiles.first, datafile, prev, next);
14
}
15
16
database/engine/journalfile.c
+3
-1
@@ -605,7 +605,9 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
605
false);
606
607
if(!vd.is_valid) {
608
- mrg_metric_release(main_mrg, metric);
608
+ if(metric)
609
+ mrg_metric_release(main_mrg, metric);
610
+
611
continue;
612
}
613
database/engine/metric.c
+212
-53
@@ -3,6 +3,10 @@
3
typedef int32_t REFCOUNT;
4
#define REFCOUNT_DELETING (-100)
5
6
+typedef enum __attribute__ ((__packed__)) {
7
+ METRIC_FLAG_HAS_RETENTION = (1 << 0),
8
+} METRIC_FLAGS;
9
+
10
struct metric {
11
uuid_t uuid; // never changes
12
Word_t section; // never changes
@@ -12,6 +16,8 @@ struct metric {
16
time_t latest_time_s_hot; // latest time of the currently collected page
17
uint32_t latest_update_every_s; //
18
pid_t writer;
19
+ METRIC_FLAGS flags;
20
+ REFCOUNT refcount;
21
SPINLOCK spinlock; // protects all variable members
22
23
// THIS IS allocated with malloc()
@@ -19,8 +25,9 @@ struct metric {
25
};
26
27
struct mrg {
28
+ ARAL *aral[MRG_PARTITIONS];
29
+
30
struct pgc_index {
23
- ARAL *aral;
31
netdata_rwlock_t rwlock;
32
Pvoid_t uuid_judy; // each UUID has a JudyL of sections (tiers)
33
} index[MRG_PARTITIONS];
@@ -95,9 +102,75 @@ static inline size_t uuid_partition(MRG *mrg __maybe_unused, uuid_t *uuid) {
102
return u[UUID_SZ - 1] % MRG_PARTITIONS;
103
}
104
98
-static METRIC *metric_add(MRG *mrg, MRG_ENTRY *entry, bool *ret) {
105
+static inline bool metric_has_retention_unsafe(MRG *mrg __maybe_unused, METRIC *metric) {
106
+ bool has_retention = (metric->first_time_s || metric->latest_time_s_clean || metric->latest_time_s_hot);
107
+
108
+ if(has_retention && !(metric->flags & METRIC_FLAG_HAS_RETENTION)) {
109
+ metric->flags |= METRIC_FLAG_HAS_RETENTION;
110
+ __atomic_add_fetch(&mrg->stats.entries_with_retention, 1, __ATOMIC_RELAXED);
111
+ }
112
+ else if(!has_retention && (metric->flags & METRIC_FLAG_HAS_RETENTION)) {
113
+ metric->flags &= ~METRIC_FLAG_HAS_RETENTION;
114
+ __atomic_sub_fetch(&mrg->stats.entries_with_retention, 1, __ATOMIC_RELAXED);
115
+ }
116
+
117
+ return has_retention;
118
+}
119
+
120
+static inline REFCOUNT metric_acquire(MRG *mrg __maybe_unused, METRIC *metric, bool having_spinlock) {
121
+ REFCOUNT refcount;
122
+
123
+ if(!having_spinlock)
124
+ netdata_spinlock_lock(&metric->spinlock);
125
+
126
+ if(unlikely(metric->refcount < 0))
127
+ fatal("METRIC: refcount is %d (negative) during acquire", metric->refcount);
128
+
129
+ refcount = ++metric->refcount;
130
+
131
+ // update its retention flags
132
+ metric_has_retention_unsafe(mrg, metric);
133
+
134
+ if(!having_spinlock)
135
+ netdata_spinlock_unlock(&metric->spinlock);
136
+
137
+ if(refcount == 1)
138
+ __atomic_add_fetch(&mrg->stats.entries_referenced, 1, __ATOMIC_RELAXED);
139
+
140
+ __atomic_add_fetch(&mrg->stats.current_references, 1, __ATOMIC_RELAXED);
141
+
142
+ return refcount;
143
+}
144
+
145
+static inline bool metric_release_and_can_be_deleted(MRG *mrg __maybe_unused, METRIC *metric) {
146
+ bool ret = true;
147
+ REFCOUNT refcount;
148
+
149
+ netdata_spinlock_lock(&metric->spinlock);
150
+
151
+ if(unlikely(metric->refcount <= 0))
152
+ fatal("METRIC: refcount is %d (zero or negative) during release", metric->refcount);
153
+
154
+ refcount = --metric->refcount;
155
+
156
+ if(likely(metric_has_retention_unsafe(mrg, metric) || refcount != 0))
157
+ ret = false;
158
+
159
+ netdata_spinlock_unlock(&metric->spinlock);
160
+
161
+ if(unlikely(!refcount))
162
+ __atomic_sub_fetch(&mrg->stats.entries_referenced, 1, __ATOMIC_RELAXED);
163
+
164
+ __atomic_sub_fetch(&mrg->stats.current_references, 1, __ATOMIC_RELAXED);
165
+
166
+ return ret;
167
+}
168
+
169
+static METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {
170
size_t partition = uuid_partition(mrg, &entry->uuid);
171
172
+ METRIC *allocation = aral_mallocz(mrg->aral[partition]);
173
+
174
mrg_index_write_lock(mrg, partition);
175
176
size_t mem_before_judyl, mem_after_judyl;
@@ -117,18 +190,22 @@ static METRIC *metric_add(MRG *mrg, MRG_ENTRY *entry, bool *ret) {
190
if(unlikely(!PValue || PValue == PJERR))
191
fatal("DBENGINE METRIC: corrupted section JudyL array");
192
120
- if(*PValue != NULL) {
193
+ if(unlikely(*PValue != NULL)) {
194
METRIC *metric = *PValue;
195
+
196
+ metric_acquire(mrg, metric, false);
197
mrg_index_write_unlock(mrg, partition);
198
199
if(ret)
200
*ret = false;
201
202
+ aral_freez(mrg->aral[partition], allocation);
203
+
204
MRG_STATS_DUPLICATE_ADD(mrg);
205
return metric;
206
}
207
131
- METRIC *metric = arrayalloc_mallocz(mrg->index[partition].aral);
208
+ METRIC *metric = allocation;
209
uuid_copy(metric->uuid, entry->uuid);
210
metric->section = entry->section;
211
metric->first_time_s = entry->first_time_s;
@@ -136,7 +213,10 @@ static METRIC *metric_add(MRG *mrg, MRG_ENTRY *entry, bool *ret) {
213
metric->latest_time_s_hot = 0;
214
metric->latest_update_every_s = entry->latest_update_every_s;
215
metric->writer = 0;
216
+ metric->refcount = 0;
217
+ metric->flags = 0;
218
netdata_spinlock_init(&metric->spinlock);
219
+ metric_acquire(mrg, metric, true); // no spinlock use required here
220
*PValue = metric;
221
222
mrg_index_write_unlock(mrg, partition);
@@ -149,7 +229,7 @@ static METRIC *metric_add(MRG *mrg, MRG_ENTRY *entry, bool *ret) {
229
return metric;
230
}
231
152
-static METRIC *metric_get(MRG *mrg, uuid_t *uuid, Word_t section) {
232
+static METRIC *metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section) {
233
size_t partition = uuid_partition(mrg, uuid);
234
235
mrg_index_read_lock(mrg, partition);
@@ -170,19 +250,27 @@ static METRIC *metric_get(MRG *mrg, uuid_t *uuid, Word_t section) {
250
251
METRIC *metric = *PValue;
252
253
+ metric_acquire(mrg, metric, false);
254
+
255
mrg_index_read_unlock(mrg, partition);
256
257
MRG_STATS_SEARCH_HIT(mrg);
258
return metric;
259
}
260
179
-static bool metric_del(MRG *mrg, METRIC *metric) {
261
+static bool acquired_metric_del(MRG *mrg, METRIC *metric) {
262
size_t partition = uuid_partition(mrg, &metric->uuid);
263
264
size_t mem_before_judyl, mem_after_judyl;
265
266
mrg_index_write_lock(mrg, partition);
267
268
+ if(!metric_release_and_can_be_deleted(mrg, metric)) {
269
+ mrg_index_write_unlock(mrg, partition);
270
+ __atomic_add_fetch(&mrg->stats.delete_having_retention_or_referenced, 1, __ATOMIC_RELAXED);
271
+ return false;
272
+ }
273
+
274
Pvoid_t *sections_judy_pptr = JudyHSGet(mrg->index[partition].uuid_judy, &metric->uuid, sizeof(uuid_t));
275
if(unlikely(!sections_judy_pptr || !*sections_judy_pptr)) {
276
mrg_index_write_unlock(mrg, partition);
@@ -208,11 +296,10 @@ static bool metric_del(MRG *mrg, METRIC *metric) {
296
mrg_stats_size_judyhs_removed_uuid(mrg);
297
}
298
211
- // arrayalloc is running lockless here
212
- arrayalloc_freez(mrg->index[partition].aral, metric);
213
-
299
mrg_index_write_unlock(mrg, partition);
300
301
+ aral_freez(mrg->aral[partition], metric);
302
+
303
MRG_STATS_DELETED_METRIC(mrg, partition);
304
305
return true;
@@ -223,11 +310,22 @@ static bool metric_del(MRG *mrg, METRIC *metric) {
310
311
MRG *mrg_create(void) {
312
MRG *mrg = callocz(1, sizeof(MRG));
313
+
314
for(size_t i = 0; i < MRG_PARTITIONS ; i++) {
315
+ char buf[ARAL_MAX_NAME + 1];
316
+ snprintfz(buf, ARAL_MAX_NAME, "mrg[%zu]", i);
317
netdata_rwlock_init(&mrg->index[i].rwlock);
228
- mrg->index[i].aral = arrayalloc_create(sizeof(METRIC), 32768 / sizeof(METRIC), NULL, NULL, false, true);
318
+
319
+ mrg->aral[i] = aral_create("mrg",
320
+ sizeof(METRIC),
321
+ 0,
322
+ 512,
323
+ NULL, NULL, false,
324
+ false);
325
}
326
+
327
mrg->stats.size = sizeof(MRG);
328
+
329
return mrg;
330
}
331
@@ -242,32 +340,27 @@ void mrg_destroy(MRG *mrg __maybe_unused) {
340
}
341
342
METRIC *mrg_metric_add_and_acquire(MRG *mrg, MRG_ENTRY entry, bool *ret) {
245
- // FIXME - support refcount
246
-
343
// internal_fatal(entry.latest_time_s > max_acceptable_collected_time(),
344
// "DBENGINE METRIC: metric latest time is in the future");
345
250
- return metric_add(mrg, &entry, ret);
346
+ return metric_add_and_acquire(mrg, &entry, ret);
347
}
348
349
METRIC *mrg_metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section) {
254
- // FIXME - support refcount
255
- return metric_get(mrg, uuid, section);
350
+ return metric_get_and_acquire(mrg, uuid, section);
351
}
352
353
bool mrg_metric_release_and_delete(MRG *mrg, METRIC *metric) {
259
- // FIXME - support refcount
260
- return metric_del(mrg, metric);
354
+ return acquired_metric_del(mrg, metric);
355
}
356
263
-METRIC *mrg_metric_dup(MRG *mrg __maybe_unused, METRIC *metric) {
264
- // FIXME - duplicate refcount
357
+METRIC *mrg_metric_dup(MRG *mrg, METRIC *metric) {
358
+ metric_acquire(mrg, metric, false);
359
return metric;
360
}
361
268
-void mrg_metric_release(MRG *mrg __maybe_unused, METRIC *metric __maybe_unused) {
269
- // FIXME - release refcount
270
-
362
+bool mrg_metric_release(MRG *mrg, METRIC *metric) {
363
+ return metric_release_and_can_be_deleted(mrg, metric);
364
}
365
366
Word_t mrg_metric_id(MRG *mrg __maybe_unused, METRIC *metric) {
@@ -285,6 +378,7 @@ Word_t mrg_metric_section(MRG *mrg __maybe_unused, METRIC *metric) {
378
bool mrg_metric_set_first_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s) {
379
netdata_spinlock_lock(&metric->spinlock);
380
metric->first_time_s = first_time_s;
381
+ metric_has_retention_unsafe(mrg, metric);
382
netdata_spinlock_unlock(&metric->spinlock);
383
384
return true;
@@ -311,6 +405,7 @@ void mrg_metric_expand_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t
405
else if(unlikely(!metric->latest_update_every_s && update_every_s))
406
metric->latest_update_every_s = update_every_s;
407
408
+ metric_has_retention_unsafe(mrg, metric);
409
netdata_spinlock_unlock(&metric->spinlock);
410
}
411
@@ -322,6 +417,7 @@ bool mrg_metric_set_first_time_s_if_bigger(MRG *mrg __maybe_unused, METRIC *metr
417
metric->first_time_s = first_time_s;
418
ret = true;
419
}
420
+ metric_has_retention_unsafe(mrg, metric);
421
netdata_spinlock_unlock(&metric->spinlock);
422
423
return ret;
@@ -382,10 +478,63 @@ bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric,
478
// if(unlikely(metric->first_time_s > latest_time_s))
479
// metric->first_time_s = latest_time_s;
480
481
+ metric_has_retention_unsafe(mrg, metric);
482
netdata_spinlock_unlock(&metric->spinlock);
483
return true;
484
}
485
486
+// returns true when metric still has retention
487
+bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metric) {
488
+ Word_t section = mrg_metric_section(mrg, metric);
489
+ bool do_again = false;
490
+ size_t countdown = 5;
491
+ bool ret = true;
492
+
493
+ do {
494
+ time_t min_first_time_s = LONG_MAX;
495
+ time_t max_end_time_s = 0;
496
+ PGC_PAGE *page;
497
+ PGC_SEARCH method = PGC_SEARCH_FIRST;
498
+ time_t page_first_time_s = 0;
499
+ time_t page_end_time_s = 0;
500
+ while ((page = pgc_page_get_and_acquire(main_cache, section, (Word_t)metric, page_first_time_s, method))) {
501
+ method = PGC_SEARCH_NEXT;
502
+
503
+ bool is_hot = pgc_is_page_hot(page);
504
+ bool is_dirty = pgc_is_page_dirty(page);
505
+ page_first_time_s = pgc_page_start_time_s(page);
506
+ page_end_time_s = pgc_page_end_time_s(page);
507
+
508
+ if ((is_hot || is_dirty) && page_first_time_s < min_first_time_s)
509
+ min_first_time_s = page_first_time_s;
510
+
511
+ if (is_dirty && page_end_time_s > max_end_time_s)
512
+ max_end_time_s = page_end_time_s;
513
+
514
+ pgc_page_release(main_cache, page);
515
+ }
516
+
517
+ if (min_first_time_s == LONG_MAX)
518
+ min_first_time_s = 0;
519
+
520
+ netdata_spinlock_lock(&metric->spinlock);
521
+ if (--countdown && !min_first_time_s && metric->latest_time_s_hot)
522
+ do_again = true;
523
+ else {
524
+ internal_error(!countdown, "METRIC: giving up on updating the retention of metric without disk retention");
525
+
526
+ do_again = false;
527
+ metric->first_time_s = min_first_time_s;
528
+ metric->latest_time_s_clean = max_end_time_s;
529
+
530
+ ret = metric_has_retention_unsafe(mrg, metric);
531
+ }
532
+ netdata_spinlock_unlock(&metric->spinlock);
533
+ } while(do_again);
534
+
535
+ return ret;
536
+}
537
+
538
bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {
539
// internal_fatal(latest_time_s > max_acceptable_collected_time(),
540
// "DBENGINE METRIC: metric latest time is in the future");
@@ -399,6 +548,7 @@ bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, t
548
// if(unlikely(metric->first_time_s > latest_time_s))
549
// metric->first_time_s = latest_time_s;
550
551
+ metric_has_retention_unsafe(mrg, metric);
552
netdata_spinlock_unlock(&metric->spinlock);
553
return true;
554
}
@@ -444,7 +594,7 @@ time_t mrg_metric_get_update_every_s(MRG *mrg __maybe_unused, METRIC *metric) {
594
return update_every_s;
595
}
596
447
-bool mrg_metric_writer_acquire(MRG *mrg, METRIC *metric) {
597
+bool mrg_metric_set_writer(MRG *mrg, METRIC *metric) {
598
bool done = false;
599
netdata_spinlock_lock(&metric->spinlock);
600
if(!metric->writer) {
@@ -452,11 +602,13 @@ bool mrg_metric_writer_acquire(MRG *mrg, METRIC *metric) {
602
__atomic_add_fetch(&mrg->stats.writers, 1, __ATOMIC_RELAXED);
603
done = true;
604
}
605
+ else
606
+ __atomic_add_fetch(&mrg->stats.writers_conflicts, 1, __ATOMIC_RELAXED);
607
netdata_spinlock_unlock(&metric->spinlock);
608
return done;
609
}
610
459
-bool mrg_metric_writer_release(MRG *mrg, METRIC *metric) {
611
+bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric) {
612
bool done = false;
613
netdata_spinlock_lock(&metric->spinlock);
614
if(metric->writer) {
@@ -584,73 +736,80 @@ static void *mrg_stress_test_thread3(void *ptr) {
736
737
int mrg_unittest(void) {
738
MRG *mrg = mrg_create();
587
- METRIC *metric1, *metric2;
739
+ METRIC *m1_t0, *m2_t0, *m3_t0, *m4_t0;
740
+ METRIC *m1_t1, *m2_t1, *m3_t1, *m4_t1;
741
bool ret;
742
743
MRG_ENTRY entry = {
591
- .section = 1,
744
+ .section = 0,
745
.first_time_s = 2,
746
.last_time_s = 3,
747
.latest_update_every_s = 4,
748
};
749
uuid_generate(entry.uuid);
597
- metric1 = mrg_metric_add_and_acquire(mrg, entry, &ret);
750
+ m1_t0 = mrg_metric_add_and_acquire(mrg, entry, &ret);
751
if(!ret)
752
fatal("DBENGINE METRIC: failed to add metric");
753
754
// add the same metric again
602
- if(mrg_metric_add_and_acquire(mrg, entry, &ret) != metric1)
755
+ m2_t0 = mrg_metric_add_and_acquire(mrg, entry, &ret);
756
+ if(m2_t0 != m1_t0)
757
fatal("DBENGINE METRIC: adding the same metric twice, does not return the same pointer");
758
if(ret)
759
fatal("DBENGINE METRIC: managed to add the same metric twice");
760
607
- if(mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section) != metric1)
761
+ m3_t0 = mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section);
762
+ if(m3_t0 != m1_t0)
763
fatal("DBENGINE METRIC: cannot find the metric added");
764
765
// add the same metric again
611
- if(mrg_metric_add_and_acquire(mrg, entry, &ret) != metric1)
766
+ m4_t0 = mrg_metric_add_and_acquire(mrg, entry, &ret);
767
+ if(m4_t0 != m1_t0)
768
fatal("DBENGINE METRIC: adding the same metric twice, does not return the same pointer");
769
if(ret)
770
fatal("DBENGINE METRIC: managed to add the same metric twice");
771
772
// add the same metric in another section
617
- entry.section = 0;
618
- metric2 = mrg_metric_add_and_acquire(mrg, entry, &ret);
773
+ entry.section = 1;
774
+ m1_t1 = mrg_metric_add_and_acquire(mrg, entry, &ret);
775
if(!ret)
620
- fatal("DBENGINE METRIC: failed to add metric in different section");
776
+ fatal("DBENGINE METRIC: failed to add metric in section %zu", (size_t)entry.section);
777
778
// add the same metric again
623
- if(mrg_metric_add_and_acquire(mrg, entry, &ret) != metric2)
624
- fatal("DBENGINE METRIC: adding the same metric twice (section 0), does not return the same pointer");
779
+ m2_t1 = mrg_metric_add_and_acquire(mrg, entry, &ret);
780
+ if(m2_t1 != m1_t1)
781
+ fatal("DBENGINE METRIC: adding the same metric twice (section %zu), does not return the same pointer", (size_t)entry.section);
782
if(ret)
783
fatal("DBENGINE METRIC: managed to add the same metric twice in (section 0)");
784
628
- if(mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section) != metric2)
629
- fatal("DBENGINE METRIC: cannot find the metric added (section 0)");
785
+ m3_t1 = mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section);
786
+ if(m3_t1 != m1_t1)
787
+ fatal("DBENGINE METRIC: cannot find the metric added (section %zu)", (size_t)entry.section);
788
789
// delete the first metric
632
- if(!mrg_metric_release_and_delete(mrg, metric1))
790
+ mrg_metric_release(mrg, m2_t0);
791
+ mrg_metric_release(mrg, m3_t0);
792
+ mrg_metric_release(mrg, m4_t0);
793
+ mrg_metric_set_first_time_s(mrg, m1_t0, 0);
794
+ mrg_metric_set_clean_latest_time_s(mrg, m1_t0, 0);
795
+ mrg_metric_set_hot_latest_time_s(mrg, m1_t0, 0);
796
+ if(!mrg_metric_release_and_delete(mrg, m1_t0))
797
fatal("DBENGINE METRIC: cannot delete the first metric");
798
635
- if(mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section) != metric2)
636
- fatal("DBENGINE METRIC: cannot find the metric added (section 0), after deleting the first one");
637
-
638
- // delete the first metric again - metric1 pointer is invalid now
639
- if(mrg_metric_release_and_delete(mrg, metric1))
640
- fatal("DBENGINE METRIC: deleted again an already deleted metric");
641
-
642
- // find the section 0 metric again
643
- if(mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section) != metric2)
644
- fatal("DBENGINE METRIC: cannot find the metric added (section 0), after deleting the first one twice");
799
+ m4_t1 = mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section);
800
+ if(m4_t1 != m1_t1)
801
+ fatal("DBENGINE METRIC: cannot find the metric added (section %zu), after deleting the first one", (size_t)entry.section);
802
803
// delete the second metric
647
- if(!mrg_metric_release_and_delete(mrg, metric2))
804
+ mrg_metric_release(mrg, m2_t1);
805
+ mrg_metric_release(mrg, m3_t1);
806
+ mrg_metric_release(mrg, m4_t1);
807
+ mrg_metric_set_first_time_s(mrg, m1_t1, 0);
808
+ mrg_metric_set_clean_latest_time_s(mrg, m1_t1, 0);
809
+ mrg_metric_set_hot_latest_time_s(mrg, m1_t1, 0);
810
+ if(!mrg_metric_release_and_delete(mrg, m1_t1))
811
fatal("DBENGINE METRIC: cannot delete the second metric");
812
650
- // delete the second metric again
651
- if(mrg_metric_release_and_delete(mrg, metric2))
652
- fatal("DBENGINE METRIC: managed to delete an already deleted metric");
653
-
813
if(mrg->stats.entries != 0)
814
fatal("DBENGINE METRIC: invalid entries counter");
815
database/engine/metric.h
+16
-6
@@ -18,23 +18,32 @@ typedef struct mrg_entry {
18
19
struct mrg_statistics {
20
size_t entries;
21
- size_t size; // memory without indexing
21
+ size_t entries_referenced;
22
+ size_t entries_with_retention;
23
+
24
+ size_t size; // total memory used, with indexing
25
+
26
+ size_t current_references;
27
+
28
size_t additions;
29
size_t additions_duplicate;
30
+
31
size_t deletions;
32
+ size_t delete_having_retention_or_referenced;
33
size_t delete_misses;
34
+
35
size_t search_hits;
36
size_t search_misses;
28
- size_t pointer_validation_hits;
29
- size_t pointer_validation_misses;
37
+
38
size_t writers;
39
+ size_t writers_conflicts;
40
};
41
42
MRG *mrg_create(void);
43
void mrg_destroy(MRG *mrg);
44
45
METRIC *mrg_metric_dup(MRG *mrg, METRIC *metric);
37
-void mrg_metric_release(MRG *mrg, METRIC *metric);
46
+bool mrg_metric_release(MRG *mrg, METRIC *metric);
47
48
METRIC *mrg_metric_add_and_acquire(MRG *mrg, MRG_ENTRY entry, bool *ret);
49
METRIC *mrg_metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section);
@@ -58,9 +67,10 @@ time_t mrg_metric_get_update_every_s(MRG *mrg, METRIC *metric);
67
68
void mrg_metric_expand_retention(MRG *mrg, METRIC *metric, time_t first_time_s, time_t last_time_s, time_t update_every_s);
69
void mrg_metric_get_retention(MRG *mrg, METRIC *metric, time_t *first_time_s, time_t *last_time_s, time_t *update_every_s);
70
+bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metric);
71
62
-bool mrg_metric_writer_acquire(MRG *mrg, METRIC *metric);
63
-bool mrg_metric_writer_release(MRG *mrg, METRIC *metric);
72
+bool mrg_metric_set_writer(MRG *mrg, METRIC *metric);
73
+bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric);
74
75
struct mrg_statistics mrg_get_statistics(MRG *mrg);
76
database/engine/pagecache.c
+8
-2
@@ -23,6 +23,9 @@ static void main_cache_flush_dirty_page_init_callback(PGC *cache __maybe_unused,
23
24
static void main_cache_flush_dirty_page_callback(PGC *cache __maybe_unused, PGC_ENTRY *entries_array __maybe_unused, PGC_PAGE **pages_array __maybe_unused, size_t entries __maybe_unused)
25
{
26
+ if(!entries)
27
+ return;
28
+
29
struct rrdengine_instance *ctx = (struct rrdengine_instance *) entries_array[0].section;
30
31
size_t bytes_per_point = CTX_POINT_SIZE_BYTES(ctx);
@@ -50,8 +53,8 @@ static void main_cache_flush_dirty_page_callback(PGC *cache __maybe_unused, PGC_
53
error_limit(&erl, "DBENGINE: page exceeds the maximum size, adjusting it to max.");
54
}
55
53
- memcpy(descr->page, pgc_page_data(pages_array[Index]), descr->page_length);
54
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(base, descr, link.prev, link.next);
56
+ descr->page = pgc_page_data(pages_array[Index]);
57
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(base, descr, link.prev, link.next);
58
59
internal_fatal(descr->page_length > RRDENG_BLOCK_SIZE, "DBENGINE: faulty page length calculation");
60
}
@@ -1074,6 +1077,7 @@ void init_page_cache(void)
1077
}
1078
1079
main_cache = pgc_create(
1080
+ "main_cache",
1081
main_cache_size,
1082
main_cache_free_clean_page_callback,
1083
(size_t) rrdeng_pages_per_extent,
@@ -1089,6 +1093,7 @@ void init_page_cache(void)
1093
);
1094
1095
open_cache = pgc_create(
1096
+ "open_cache",
1097
open_cache_size, // the default is 1MB
1098
open_cache_free_clean_page_callback,
1099
1,
@@ -1105,6 +1110,7 @@ void init_page_cache(void)
1110
pgc_set_dynamic_target_cache_size_callback(open_cache, dynamic_open_cache_size);
1111
1112
extent_cache = pgc_create(
1113
+ "extent_cache",
1114
extent_cache_size,
1115
extent_cache_free_clean_page_callback,
1116
1,
database/engine/pagecache.h
+1
-1
@@ -27,7 +27,7 @@ struct page_descr_with_data {
27
uint8_t type;
28
uint32_t update_every_s;
29
uint32_t page_length;
30
- uint8_t page[RRDENG_BLOCK_SIZE];
30
+ uint8_t *page;
31
32
struct {
33
struct page_descr_with_data *prev;
database/engine/pdc.c
+50
-27
@@ -68,7 +68,7 @@ void pdc_cleanup1(void) {
68
69
if(pdc_globals.protected.available_items && pdc_globals.protected.available > (size_t)libuv_worker_threads) {
70
item = pdc_globals.protected.available_items;
71
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(pdc_globals.protected.available_items, item, cache.prev, cache.next);
71
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(pdc_globals.protected.available_items, item, cache.prev, cache.next);
72
pdc_globals.protected.available--;
73
}
74
@@ -87,7 +87,7 @@ PDC *pdc_get(void) {
87
88
if(likely(pdc_globals.protected.available_items)) {
89
pdc = pdc_globals.protected.available_items;
90
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(pdc_globals.protected.available_items, pdc, cache.prev, cache.next);
90
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(pdc_globals.protected.available_items, pdc, cache.prev, cache.next);
91
pdc_globals.protected.available--;
92
}
93
@@ -106,7 +106,7 @@ static void pdc_release(PDC *pdc) {
106
if(unlikely(!pdc)) return;
107
108
netdata_spinlock_lock(&pdc_globals.protected.spinlock);
109
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(pdc_globals.protected.available_items, pdc, cache.prev, cache.next);
109
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(pdc_globals.protected.available_items, pdc, cache.prev, cache.next);
110
pdc_globals.protected.available++;
111
netdata_spinlock_unlock(&pdc_globals.protected.spinlock);
112
}
@@ -147,7 +147,7 @@ void page_details_cleanup1(void) {
147
148
if(page_details_globals.protected.available_items && page_details_globals.protected.available > (size_t)libuv_worker_threads * 2) {
149
item = page_details_globals.protected.available_items;
150
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(page_details_globals.protected.available_items, item, cache.prev, cache.next);
150
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_details_globals.protected.available_items, item, cache.prev, cache.next);
151
page_details_globals.protected.available--;
152
}
153
@@ -166,7 +166,7 @@ struct page_details *page_details_get(void) {
166
167
if(likely(page_details_globals.protected.available_items)) {
168
pd = page_details_globals.protected.available_items;
169
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(page_details_globals.protected.available_items, pd, cache.prev, cache.next);
169
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_details_globals.protected.available_items, pd, cache.prev, cache.next);
170
page_details_globals.protected.available--;
171
}
172
@@ -185,7 +185,7 @@ static void page_details_release(struct page_details *pd) {
185
if(unlikely(!pd)) return;
186
187
netdata_spinlock_lock(&page_details_globals.protected.spinlock);
188
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(page_details_globals.protected.available_items, pd, cache.prev, cache.next);
188
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(page_details_globals.protected.available_items, pd, cache.prev, cache.next);
189
page_details_globals.protected.available++;
190
netdata_spinlock_unlock(&page_details_globals.protected.spinlock);
191
}
@@ -226,7 +226,7 @@ void epdl_cleanup1(void) {
226
227
if(epdl_globals.protected.available_items && epdl_globals.protected.available > 100) {
228
item = epdl_globals.protected.available_items;
229
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(epdl_globals.protected.available_items, item, cache.prev, cache.next);
229
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(epdl_globals.protected.available_items, item, cache.prev, cache.next);
230
epdl_globals.protected.available--;
231
}
232
@@ -245,7 +245,7 @@ static EPDL *epdl_get(void) {
245
246
if(likely(epdl_globals.protected.available_items)) {
247
epdl = epdl_globals.protected.available_items;
248
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(epdl_globals.protected.available_items, epdl, cache.prev, cache.next);
248
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(epdl_globals.protected.available_items, epdl, cache.prev, cache.next);
249
epdl_globals.protected.available--;
250
}
251
@@ -264,7 +264,7 @@ static void epdl_release(EPDL *epdl) {
264
if(unlikely(!epdl)) return;
265
266
netdata_spinlock_lock(&epdl_globals.protected.spinlock);
267
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(epdl_globals.protected.available_items, epdl, cache.prev, cache.next);
267
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(epdl_globals.protected.available_items, epdl, cache.prev, cache.next);
268
epdl_globals.protected.available++;
269
netdata_spinlock_unlock(&epdl_globals.protected.spinlock);
270
}
@@ -305,7 +305,7 @@ void deol_cleanup1(void) {
305
306
if(deol_globals.protected.available_items && deol_globals.protected.available > 100) {
307
item = deol_globals.protected.available_items;
308
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(deol_globals.protected.available_items, item, cache.prev, cache.next);
308
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(deol_globals.protected.available_items, item, cache.prev, cache.next);
309
deol_globals.protected.available--;
310
}
311
@@ -324,7 +324,7 @@ static DEOL *deol_get(void) {
324
325
if(likely(deol_globals.protected.available_items)) {
326
deol = deol_globals.protected.available_items;
327
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(deol_globals.protected.available_items, deol, cache.prev, cache.next);
327
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(deol_globals.protected.available_items, deol, cache.prev, cache.next);
328
deol_globals.protected.available--;
329
}
330
@@ -343,7 +343,7 @@ static void deol_release(DEOL *deol) {
343
if(unlikely(!deol)) return;
344
345
netdata_spinlock_lock(&deol_globals.protected.spinlock);
346
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(deol_globals.protected.available_items, deol, cache.prev, cache.next);
346
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(deol_globals.protected.available_items, deol, cache.prev, cache.next);
347
deol_globals.protected.available++;
348
netdata_spinlock_unlock(&deol_globals.protected.spinlock);
349
}
@@ -399,7 +399,7 @@ void extent_buffer_cleanup1(void) {
399
400
if(extent_buffer_globals.protected.available_items && extent_buffer_globals.protected.available > 1) {
401
item = extent_buffer_globals.protected.available_items;
402
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(extent_buffer_globals.protected.available_items, item, cache.prev, cache.next);
402
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(extent_buffer_globals.protected.available_items, item, cache.prev, cache.next);
403
extent_buffer_globals.protected.available--;
404
}
405
@@ -424,7 +424,7 @@ struct extent_buffer *extent_buffer_get(size_t size) {
424
netdata_spinlock_lock(&extent_buffer_globals.protected.spinlock);
425
if(likely(extent_buffer_globals.protected.available_items)) {
426
eb = extent_buffer_globals.protected.available_items;
427
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(extent_buffer_globals.protected.available_items, eb, cache.prev, cache.next);
427
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(extent_buffer_globals.protected.available_items, eb, cache.prev, cache.next);
428
extent_buffer_globals.protected.available--;
429
}
430
netdata_spinlock_unlock(&extent_buffer_globals.protected.spinlock);
@@ -452,7 +452,7 @@ void extent_buffer_release(struct extent_buffer *eb) {
452
if(unlikely(!eb)) return;
453
454
netdata_spinlock_lock(&extent_buffer_globals.protected.spinlock);
455
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(extent_buffer_globals.protected.available_items, eb, cache.prev, cache.next);
455
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(extent_buffer_globals.protected.available_items, eb, cache.prev, cache.next);
456
extent_buffer_globals.protected.available++;
457
netdata_spinlock_unlock(&extent_buffer_globals.protected.spinlock);
458
}
@@ -671,7 +671,7 @@ static bool epdl_pending_add(EPDL *epdl) {
671
rrdeng_req_cmd(epdl_get_cmd, base, epdl->pdc->priority);
672
}
673
674
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(base, epdl, query.prev, query.next);
674
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(base, epdl, query.prev, query.next);
675
*PValue = base;
676
677
netdata_spinlock_unlock(&epdl->datafile->extent_queries.spinlock);
@@ -1009,7 +1009,7 @@ static inline struct page_details *epdl_get_pd_load_link_list_from_metric_start_
1009
if (unlikely(__atomic_load_n(&ep->pdc->workers_should_stop, __ATOMIC_RELAXED)))
1010
pdc_page_status_set(pd, PDC_PAGE_FAILED | PDC_PAGE_CANCELLED);
1011
else
1012
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(pd_list, pd, load.prev, load.next);
1012
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(pd_list, pd, load.prev, load.next);
1013
}
1014
}
1015
}
@@ -1259,6 +1259,35 @@ static bool epdl_populate_pages_from_extent_data(
1259
return true;
1260
}
1261
1262
+static inline void *datafile_extent_read(struct rrdengine_instance *ctx, uv_file file, unsigned pos, unsigned size_bytes)
1263
+{
1264
+ void *buffer;
1265
+ uv_fs_t request;
1266
+
1267
+ unsigned real_io_size = ALIGN_BYTES_CEILING(size_bytes);
1268
+ int ret = posix_memalign(&buffer, RRDFILE_ALIGNMENT, real_io_size);
1269
+ if (unlikely(ret))
1270
+ fatal("DBENGINE: posix_memalign(): %s", strerror(ret));
1271
+
1272
+ uv_buf_t iov = uv_buf_init(buffer, real_io_size);
1273
+ ret = uv_fs_read(NULL, &request, file, &iov, 1, pos, NULL);
1274
+ if (unlikely(-1 == ret)) {
1275
+ ctx_io_error(ctx);
1276
+ posix_memfree(buffer);
1277
+ buffer = NULL;
1278
+ }
1279
+ else
1280
+ ctx_io_read_op_bytes(ctx, real_io_size);
1281
+
1282
+ uv_fs_req_cleanup(&request);
1283
+
1284
+ return buffer;
1285
+}
1286
+
1287
+static inline void datafile_extent_read_free(void *buffer) {
1288
+ posix_memfree(buffer);
1289
+}
1290
+
1291
void epdl_find_extent_and_populate_pages(struct rrdengine_instance *ctx, EPDL *epdl, bool worker) {
1292
size_t *statistics_counter = NULL;
1293
PDC_PAGE_STATUS not_loaded_pages_tag = 0, loaded_pages_tag = 0;
@@ -1306,18 +1335,12 @@ void epdl_find_extent_and_populate_pages(struct rrdengine_instance *ctx, EPDL *e
1335
if(worker)
1336
worker_is_busy(UV_EVENT_DBENGINE_EXTENT_MMAP);
1337
1309
- off_t map_start = ALIGN_BYTES_FLOOR(epdl->extent_offset);
1310
- size_t length = ALIGN_BYTES_CEILING(epdl->extent_offset + epdl->extent_size) - map_start;
1311
-
1312
- void *mmap_data = mmap(NULL, length, PROT_READ, MAP_SHARED, epdl->file, map_start);
1313
- if(mmap_data != MAP_FAILED) {
1314
- extent_compressed_data = mmap_data + (epdl->extent_offset - map_start);
1338
+ void *extent_data = datafile_extent_read(ctx, epdl->file, epdl->extent_offset, epdl->extent_size);
1339
+ if(extent_data != NULL) {
1340
1341
void *copied_extent_compressed_data = dbengine_extent_alloc(epdl->extent_size);
1317
- memcpy(copied_extent_compressed_data, extent_compressed_data, epdl->extent_size);
1318
-
1319
- int ret = munmap(mmap_data, length);
1320
- fatal_assert(0 == ret);
1342
+ memcpy(copied_extent_compressed_data, extent_data, epdl->extent_size);
1343
+ datafile_extent_read_free(extent_data);
1344
1345
if(worker)
1346
worker_is_busy(UV_EVENT_DBENGINE_EXTENT_CACHE_LOOKUP);
database/engine/rrdengine.c
+58
-33
@@ -124,7 +124,7 @@ static void work_request_cleanup1(void) {
124
125
if(work_request_globals.protected.available_items && work_request_globals.protected.available > (size_t)libuv_worker_threads) {
126
item = work_request_globals.protected.available_items;
127
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(work_request_globals.protected.available_items, item, cache.prev, cache.next);
127
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(work_request_globals.protected.available_items, item, cache.prev, cache.next);
128
work_request_globals.protected.available--;
129
}
130
netdata_spinlock_unlock(&work_request_globals.protected.spinlock);
@@ -137,7 +137,7 @@ static void work_request_cleanup1(void) {
137
138
static inline void work_done(struct rrdeng_work *work_request) {
139
netdata_spinlock_lock(&work_request_globals.protected.spinlock);
140
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(work_request_globals.protected.available_items, work_request, cache.prev, cache.next);
140
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(work_request_globals.protected.available_items, work_request, cache.prev, cache.next);
141
work_request_globals.protected.available++;
142
netdata_spinlock_unlock(&work_request_globals.protected.spinlock);
143
}
@@ -183,7 +183,7 @@ static bool work_dispatch(struct rrdengine_instance *ctx, void *data, struct com
183
184
if(likely(work_request_globals.protected.available_items)) {
185
work_request = work_request_globals.protected.available_items;
186
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(work_request_globals.protected.available_items, work_request, cache.prev, cache.next);
186
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(work_request_globals.protected.available_items, work_request, cache.prev, cache.next);
187
work_request_globals.protected.available--;
188
}
189
@@ -246,7 +246,7 @@ static void page_descriptor_cleanup1(void) {
246
247
if(page_descriptor_globals.protected.available_items && page_descriptor_globals.protected.available > MAX_PAGES_PER_EXTENT) {
248
item = page_descriptor_globals.protected.available_items;
249
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(page_descriptor_globals.protected.available_items, item, cache.prev, cache.next);
249
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_descriptor_globals.protected.available_items, item, cache.prev, cache.next);
250
page_descriptor_globals.protected.available--;
251
}
252
@@ -265,7 +265,7 @@ struct page_descr_with_data *page_descriptor_get(void) {
265
266
if(likely(page_descriptor_globals.protected.available_items)) {
267
descr = page_descriptor_globals.protected.available_items;
268
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(page_descriptor_globals.protected.available_items, descr, cache.prev, cache.next);
268
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_descriptor_globals.protected.available_items, descr, cache.prev, cache.next);
269
page_descriptor_globals.protected.available--;
270
}
271
@@ -284,7 +284,7 @@ static inline void page_descriptor_release(struct page_descr_with_data *descr) {
284
if(unlikely(!descr)) return;
285
286
netdata_spinlock_lock(&page_descriptor_globals.protected.spinlock);
287
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(page_descriptor_globals.protected.available_items, descr, cache.prev, cache.next);
287
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(page_descriptor_globals.protected.available_items, descr, cache.prev, cache.next);
288
page_descriptor_globals.protected.available++;
289
netdata_spinlock_unlock(&page_descriptor_globals.protected.spinlock);
290
}
@@ -322,7 +322,7 @@ static void extent_io_descriptor_cleanup1(void) {
322
323
if(extent_io_descriptor_globals.protected.available_items && extent_io_descriptor_globals.protected.available > (size_t)libuv_worker_threads) {
324
item = extent_io_descriptor_globals.protected.available_items;
325
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(extent_io_descriptor_globals.protected.available_items, item, cache.prev, cache.next);
325
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(extent_io_descriptor_globals.protected.available_items, item, cache.prev, cache.next);
326
extent_io_descriptor_globals.protected.available--;
327
}
328
netdata_spinlock_unlock(&extent_io_descriptor_globals.protected.spinlock);
@@ -340,7 +340,7 @@ static struct extent_io_descriptor *extent_io_descriptor_get(void) {
340
341
if(likely(extent_io_descriptor_globals.protected.available_items)) {
342
xt_io_descr = extent_io_descriptor_globals.protected.available_items;
343
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(extent_io_descriptor_globals.protected.available_items, xt_io_descr, cache.prev, cache.next);
343
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(extent_io_descriptor_globals.protected.available_items, xt_io_descr, cache.prev, cache.next);
344
extent_io_descriptor_globals.protected.available--;
345
}
346
@@ -359,7 +359,7 @@ static inline void extent_io_descriptor_release(struct extent_io_descriptor *xt_
359
if(unlikely(!xt_io_descr)) return;
360
361
netdata_spinlock_lock(&extent_io_descriptor_globals.protected.spinlock);
362
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(extent_io_descriptor_globals.protected.available_items, xt_io_descr, cache.prev, cache.next);
362
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(extent_io_descriptor_globals.protected.available_items, xt_io_descr, cache.prev, cache.next);
363
extent_io_descriptor_globals.protected.available++;
364
netdata_spinlock_unlock(&extent_io_descriptor_globals.protected.spinlock);
365
}
@@ -396,7 +396,7 @@ static void rrdeng_query_handle_cleanup1(void) {
396
397
if(rrdeng_query_handle_globals.protected.available_items && rrdeng_query_handle_globals.protected.available > 10) {
398
item = rrdeng_query_handle_globals.protected.available_items;
399
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rrdeng_query_handle_globals.protected.available_items, item, cache.prev, cache.next);
399
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_query_handle_globals.protected.available_items, item, cache.prev, cache.next);
400
rrdeng_query_handle_globals.protected.available--;
401
}
402
@@ -415,7 +415,7 @@ struct rrdeng_query_handle *rrdeng_query_handle_get(void) {
415
416
if(likely(rrdeng_query_handle_globals.protected.available_items)) {
417
handle = rrdeng_query_handle_globals.protected.available_items;
418
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rrdeng_query_handle_globals.protected.available_items, handle, cache.prev, cache.next);
418
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_query_handle_globals.protected.available_items, handle, cache.prev, cache.next);
419
rrdeng_query_handle_globals.protected.available--;
420
}
421
@@ -434,7 +434,7 @@ void rrdeng_query_handle_release(struct rrdeng_query_handle *handle) {
434
if(unlikely(!handle)) return;
435
436
netdata_spinlock_lock(&rrdeng_query_handle_globals.protected.spinlock);
437
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(rrdeng_query_handle_globals.protected.available_items, handle, cache.prev, cache.next);
437
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_query_handle_globals.protected.available_items, handle, cache.prev, cache.next);
438
rrdeng_query_handle_globals.protected.available++;
439
netdata_spinlock_unlock(&rrdeng_query_handle_globals.protected.spinlock);
440
}
@@ -471,7 +471,7 @@ static void wal_cleanup1(void) {
471
472
if(wal_globals.protected.available_items && wal_globals.protected.available > storage_tiers) {
473
wal = wal_globals.protected.available_items;
474
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
474
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
475
wal_globals.protected.available--;
476
}
477
@@ -494,7 +494,7 @@ WAL *wal_get(struct rrdengine_instance *ctx, unsigned size) {
494
495
if(likely(wal_globals.protected.available_items)) {
496
wal = wal_globals.protected.available_items;
497
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
497
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
498
wal_globals.protected.available--;
499
}
500
@@ -532,7 +532,7 @@ void wal_release(WAL *wal) {
532
if(unlikely(!wal)) return;
533
534
netdata_spinlock_lock(&wal_globals.protected.spinlock);
535
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
535
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);
536
wal_globals.protected.available++;
537
netdata_spinlock_unlock(&wal_globals.protected.spinlock);
538
}
@@ -596,7 +596,7 @@ static void rrdeng_cmd_cleanup1(void) {
596
597
if(rrdeng_cmd_globals.cache.available_items && rrdeng_cmd_globals.cache.available > 100) {
598
item = rrdeng_cmd_globals.cache.available_items;
599
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rrdeng_cmd_globals.cache.available_items, item, cache.prev, cache.next);
599
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.cache.available_items, item, cache.prev, cache.next);
600
rrdeng_cmd_globals.cache.available--;
601
}
602
netdata_spinlock_unlock(&rrdeng_cmd_globals.cache.spinlock);
@@ -639,8 +639,8 @@ void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY
639
priority = rrdeng_enq_cmd_map_opcode_to_priority(cmd->opcode, priority);
640
641
if (cmd->priority > priority) {
642
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[cmd->priority], cmd, cache.prev, cache.next);
643
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
642
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[cmd->priority], cmd, cache.prev, cache.next);
643
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
644
cmd->priority = priority;
645
}
646
}
@@ -657,7 +657,7 @@ void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, v
657
netdata_spinlock_lock(&rrdeng_cmd_globals.cache.spinlock);
658
if(likely(rrdeng_cmd_globals.cache.available_items)) {
659
cmd = rrdeng_cmd_globals.cache.available_items;
660
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rrdeng_cmd_globals.cache.available_items, cmd, cache.prev, cache.next);
660
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.cache.available_items, cmd, cache.prev, cache.next);
661
rrdeng_cmd_globals.cache.available--;
662
}
663
netdata_spinlock_unlock(&rrdeng_cmd_globals.cache.spinlock);
@@ -676,7 +676,7 @@ void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, v
676
cmd->dequeue_cb = dequeue_cb;
677
678
netdata_spinlock_lock(&rrdeng_cmd_globals.queue.spinlock);
679
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
679
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
680
rrdeng_cmd_globals.queue.waiting++;
681
if(enqueue_cb)
682
enqueue_cb(cmd);
@@ -715,7 +715,7 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
715
}
716
717
// remove it from the queue
718
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
718
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
719
rrdeng_cmd_globals.queue.waiting--;
720
break;
721
}
@@ -735,7 +735,7 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
735
736
// put it in the cache
737
netdata_spinlock_lock(&rrdeng_cmd_globals.cache.spinlock);
738
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(rrdeng_cmd_globals.cache.available_items, cmd, cache.prev, cache.next);
738
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_cmd_globals.cache.available_items, cmd, cache.prev, cache.next);
739
rrdeng_cmd_globals.cache.available++;
740
netdata_spinlock_unlock(&rrdeng_cmd_globals.cache.spinlock);
741
}
@@ -968,10 +968,6 @@ static void *extent_flushed_to_open_tp_worker(struct rrdengine_instance *ctx __m
968
struct rrdengine_datafile *datafile;
969
unsigned i;
970
971
- if (uv_fs_request->result < 0) {
972
- ctx_io_error(ctx);
973
- error("DBENGINE: %s: uv_fs_write: %s", __func__, uv_strerror((int)uv_fs_request->result));
974
- }
971
datafile = xt_io_descr->datafile;
972
973
bool still_running = ctx_is_available_for_queries(ctx);
@@ -1014,6 +1010,11 @@ static void after_extent_write_datafile_io(uv_fs_t *uv_fs_request) {
1010
struct rrdengine_datafile *datafile = xt_io_descr->datafile;
1011
struct rrdengine_instance *ctx = datafile->ctx;
1012
1013
+ if (uv_fs_request->result < 0) {
1014
+ ctx_io_error(ctx);
1015
+ error("DBENGINE: %s: uv_fs_write(): %s", __func__, uv_strerror((int)uv_fs_request->result));
1016
+ }
1017
+
1018
journalfile_v1_extent_write(ctx, xt_io_descr->datafile, xt_io_descr->wal, &rrdeng_main.loop);
1019
1020
netdata_spinlock_lock(&datafile->writers.spinlock);
@@ -1398,7 +1399,7 @@ void find_uuid_first_time(
1399
}
1400
}
1401
internal_error(true,
1401
- "DBENGINE: analyzed the retention of %zu rotated metrics, "
1402
+ "DBENGINE: analyzed the retention of %zu rotated metrics of tier %d, "
1403
"did %zu jv2 matching binary searches (%zu not matching, %zu overflown) in %u journal files, "
1404
"%zu metrics with entries in open cache, "
1405
"metrics first time found per datafile index ([not in jv2]:%zu, [1]:%zu, [2]:%zu, [3]:%zu, [4]:%zu, [5]:%zu, [6]:%zu, [7]:%zu, [8]:%zu, [bigger]: %zu), "
@@ -1406,6 +1407,7 @@ void find_uuid_first_time(
1407
"metrics without any remaining retention %zu, "
1408
"metrics not in MRG %zu",
1409
metric_count,
1410
+ ctx->config.tier,
1411
binary_match,
1412
not_matching_bsearches,
1413
not_needed_bsearches,
@@ -1446,7 +1448,8 @@ static void update_metrics_first_time_s(struct rrdengine_instance *ctx, struct r
1448
added++;
1449
}
1450
1449
- info("DBENGINE: recalculating retention for %zu metrics starting with datafile %u", count, first_datafile_remaining->fileno);
1451
+ info("DBENGINE: recalculating tier %d retention for %zu metrics starting with datafile %u",
1452
+ ctx->config.tier, count, first_datafile_remaining->fileno);
1453
1454
journalfile_v2_data_release(journalfile);
1455
@@ -1460,18 +1463,40 @@ static void update_metrics_first_time_s(struct rrdengine_instance *ctx, struct r
1463
if(worker)
1464
worker_is_busy(UV_EVENT_DBENGINE_POPULATE_MRG);
1465
1463
- info("DBENGINE: updating metric registry retention for %zu metrics", added);
1466
+ info("DBENGINE: updating tier %d metrics registry retention for %zu metrics",
1467
+ ctx->config.tier, added);
1468
1469
+ size_t deleted_metrics = 0, zero_retention_referenced = 0, zero_disk_retention = 0, zero_disk_but_live = 0;
1470
for (size_t index = 0; index < added; ++index) {
1471
uuid_first_t_entry = &uuid_first_entry_list[index];
1467
- if (likely(uuid_first_t_entry->first_time_s != LONG_MAX))
1472
+ if (likely(uuid_first_t_entry->first_time_s != LONG_MAX)) {
1473
mrg_metric_set_first_time_s_if_bigger(main_mrg, uuid_first_t_entry->metric, uuid_first_t_entry->first_time_s);
1469
- else
1470
- mrg_metric_set_first_time_s(main_mrg, uuid_first_t_entry->metric, 0);
1471
- mrg_metric_release(main_mrg, uuid_first_t_entry->metric);
1474
+ mrg_metric_release(main_mrg, uuid_first_t_entry->metric);
1475
+ }
1476
+ else {
1477
+ zero_disk_retention++;
1478
+
1479
+ // there is no retention for this metric
1480
+ bool has_retention = mrg_metric_zero_disk_retention(main_mrg, uuid_first_t_entry->metric);
1481
+ if (!has_retention) {
1482
+ bool deleted = mrg_metric_release_and_delete(main_mrg, uuid_first_t_entry->metric);
1483
+ if(deleted)
1484
+ deleted_metrics++;
1485
+ else
1486
+ zero_retention_referenced++;
1487
+ }
1488
+ else {
1489
+ zero_disk_but_live++;
1490
+ mrg_metric_release(main_mrg, uuid_first_t_entry->metric);
1491
+ }
1492
+ }
1493
}
1494
freez(uuid_first_entry_list);
1495
1496
+ internal_error(zero_disk_retention,
1497
+ "DBENGINE: deleted %zu metrics, zero retention but referenced %zu (out of %zu total, of which %zu have main cache retention) zero on-disk retention tier %d metrics from metrics registry",
1498
+ deleted_metrics, zero_retention_referenced, zero_disk_retention, zero_disk_but_live, ctx->config.tier);
1499
+
1500
if(worker)
1501
worker_is_idle();
1502
}
database/engine/rrdengineapi.c
+4
-4
@@ -243,7 +243,7 @@ STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metri
243
struct rrdengine_instance *ctx = mrg_metric_ctx(metric);
244
245
bool is_1st_metric_writer = true;
246
- if(!mrg_metric_writer_acquire(main_mrg, metric)) {
246
+ if(!mrg_metric_set_writer(main_mrg, metric)) {
247
is_1st_metric_writer = false;
248
char uuid[UUID_STR_LEN + 1];
249
uuid_unparse(*mrg_metric_uuid(main_mrg, metric), uuid);
@@ -696,7 +696,7 @@ int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
696
if(!(handle->options & RRDENG_1ST_METRIC_WRITER))
697
__atomic_sub_fetch(&ctx->atomic.collectors_running_duplicate, 1, __ATOMIC_RELAXED);
698
699
- if((handle->options & RRDENG_1ST_METRIC_WRITER) && !mrg_metric_writer_release(main_mrg, handle->metric))
699
+ if((handle->options & RRDENG_1ST_METRIC_WRITER) && !mrg_metric_clear_writer(main_mrg, handle->metric))
700
internal_fatal(true, "DBENGINE: metric is already released");
701
702
time_t first_time_s, last_time_s, update_every_s;
@@ -738,12 +738,12 @@ static void register_query_handle(struct rrdeng_query_handle *handle) {
738
handle->started_time_s = now_realtime_sec();
739
740
netdata_spinlock_lock(&global_query_handle_spinlock);
741
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(global_query_handle_ll, handle, prev, next);
741
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(global_query_handle_ll, handle, prev, next);
742
netdata_spinlock_unlock(&global_query_handle_spinlock);
743
}
744
static void unregister_query_handle(struct rrdeng_query_handle *handle) {
745
netdata_spinlock_lock(&global_query_handle_spinlock);
746
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(global_query_handle_ll, handle, prev, next);
746
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(global_query_handle_ll, handle, prev, next);
747
netdata_spinlock_unlock(&global_query_handle_spinlock);
748
}
749
#else
database/rrd.h
+3
-1
@@ -436,8 +436,10 @@ void rrddim_memory_file_save(RRDDIM *rd);
436
(x).end_time_s = end_s; \
437
} while(0)
438
439
+#define STORAGE_POINT_UNSET { .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }
440
+
441
#define storage_point_is_unset(x) (!(x).count)
440
-#define storage_point_is_empty(x) (!netdata_double_isnumber((x).sum))
442
+#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))
443
444
// ------------------------------------------------------------------------
445
// function pointers that handle data collection
database/rrdcalc.c
+2
-2
@@ -181,7 +181,7 @@ static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) {
181
rc->rrdset = st;
182
183
netdata_rwlock_wrlock(&st->alerts.rwlock);
184
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->alerts.base, rc, prev, next);
184
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(st->alerts.base, rc, prev, next);
185
netdata_rwlock_unlock(&st->alerts.rwlock);
186
187
if(rc->update_every < rc->rrdset->update_every) {
@@ -328,7 +328,7 @@ static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) {
328
if(!having_ll_wrlock)
329
netdata_rwlock_wrlock(&st->alerts.rwlock);
330
331
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alerts.base, rc, prev, next);
331
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(st->alerts.base, rc, prev, next);
332
333
if(!having_ll_wrlock)
334
netdata_rwlock_unlock(&st->alerts.rwlock);
database/rrdcontext.h
+2
@@ -122,6 +122,8 @@ typedef struct query_plan_entry {
122
size_t tier;
123
time_t after;
124
time_t before;
125
+ time_t expanded_after;
126
+ time_t expanded_before;
127
struct storage_engine_query_handle handle;
128
STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
129
int (*is_finished)(struct storage_engine_query_handle *handle);
database/rrdhost.c
+3
-3
@@ -477,9 +477,9 @@ int is_legacy = 1;
477
rrdhost_index_add_hostname(host);
478
479
if(is_localhost)
480
- DOUBLE_LINKED_LIST_PREPEND_UNSAFE(localhost, host, prev, next);
480
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(localhost, host, prev, next);
481
else
482
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(localhost, host, prev, next);
482
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(localhost, host, prev, next);
483
484
rrd_unlock();
485
@@ -1092,7 +1092,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1092
rrdhost_index_del_by_guid(host);
1093
1094
if (host->prev)
1095
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(localhost, host, prev, next);
1095
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(localhost, host, prev, next);
1096
}
1097
1098
// ------------------------------------------------------------------------
database/rrdset.c
+1
-1
@@ -1149,7 +1149,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1149
t->virtual_point.end_time_s = sp.end_time_s;
1150
1151
// merge the values into our virtual point
1152
- if (likely(!storage_point_is_empty(sp))) {
1152
+ if (likely(!storage_point_is_gap(sp))) {
1153
// we aggregate only non NULLs into higher tiers
1154
1155
if (likely(!storage_point_is_unset(t->virtual_point))) {
exporting/process_data.c
+1
-1
@@ -130,7 +130,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
130
STORAGE_POINT sp = rd->tiers[0].query_ops->next_metric(&handle);
131
points_read++;
132
133
- if (unlikely(storage_point_is_empty(sp))) {
133
+ if (unlikely(storage_point_is_gap(sp))) {
134
// not collected
135
continue;
136
}
libnetdata/Makefile.am
+1
-1
@@ -5,7 +5,7 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
6
SUBDIRS = \
7
adaptive_resortable_list \
8
- arrayalloc \
8
+ aral \
9
avl \
10
buffer \
11
clocks \
libnetdata/aral/Makefile.am
renamed
libnetdata/aral/README.md
new
+169
@@ -0,0 +1,169 @@
1
+<!--
2
+title: "Array Allocator"
3
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/aral/README.md
4
+-->
5
+
6
+# Array Allocator
7
+
8
+Come on! Array allocators are embedded in libc! Why do we need such a thing in Netdata?
9
+
10
+Well, we have a couple of problems to solve:
11
+
12
+1. **Fragmentation** - It is important for Netdata to keeps its overall memory footprint as low as possible. libc does an amazing job when the same thread allocates and frees some memory. But it simply cannot do better without knowing the specifics of the application when memory is allocated and freed randomly between threads.
13
+2. **Speed** - Especially when allocations and de-allocations happen across threads, the speed penalty is tremendous.
14
+
15
+In Netdata we have a few moments that are very tough. Imagine collecting 1 million metrics per second. You have a buffer for each metric and put append new points there. This works beautifully, of course! But then, when the buffers get full, imagine the situation. You suddenly need 1 million buffers, at once!
16
+
17
+To solve this problem we first spread out the buffers. So, the first time each metric asks for a buffer, it gets a smaller one. We added logic there to spread them as evenly as possible across time. Solved? Not exactly!
18
+
19
+We have 3 tiers for each metric. For the metrics of tier 0 (per second resolution) we have a max buffer for 1024 points and every new metrics gets a random size between 3 points and 1024. So they are distributed across time. For 1 million metrics, we have about 1000 buffers beings created every second.
20
+
21
+But at some point, the end of the minute will come, and suddenly all the metrics will need a new buffer for tier 1 (per minute). Oops! We will spread tier 1 buffers across time too, but the first minute is a tough one. We really need 1 million buffers instantly.
22
+
23
+And if that minute happens to also be the beginning of an hour... tier 2 (per hour) kicks in. For that instant we are going to need 2 million buffers instantly.
24
+
25
+The problem becomes even bigger when we collect 2, or even 10 million metrics...
26
+
27
+So solve it, Netdata uses a special implementation of an array allocator that is tightly integrated with the structures we need.
28
+
29
+## Features
30
+
31
+1. Malloc, or MMAP modes. File based MMAP is also supported to put the data in file backed up shared memory.
32
+2. Fully asynchronous operations. There are just a couple of points where spin-locks protect a few counters and pointers.
33
+3. Optional defragmenter, that once enabled it will make free operation slower while trying to maintain a sorted list of fragments to offer first during allocations. The defragmenter can be enabled / disabled at run time. The defragmenter can hurt performance on application with intense turn-around of allocation, like Netdata dbengine caches. So, it is disabled by default.
34
+4. Without the defragmenter enabled, ARAL still tries to keep pages full, but the depth of the search is limited to 3 pages (so, a page with a free slot will either become 1st, 2nd, or 3rd). At the same time, during allocations, ARAL will evaluate the first 2 pages to find the one that is more full than the other, to use it for the new allocation.
35
+
36
+## How it works
37
+
38
+Allocations are organized in pages. Pages have a minimum size (a system page, usually 4KB) and a maximum defined by for each different kind of object.
39
+
40
+Initially every page is free. When an allocation request is made, the free space is split, and the first element is reserved. Free space is now considered there rest.
41
+
42
+This continuous until the page gets full, where a new page is allocated and the process is repeated.
43
+
44
+Each allocation returned has a pointer appended to it. The pointer points to the page the allocation belongs to.
45
+
46
+When a pointer is freed, the page it belongs is identified, its space is marked free, and it is prepended in a single linked list that resides in the page itself. So, each page has its own list of free slots to use.
47
+
48
+Pages are then on another linked list. This is a double linked list and at its beginning has the pages with free space and at the end the pages that are full.
49
+
50
+When the defragmenter is enabled the pages double linked list is also sorted, like this: the fewer the free slots on a page, the earlier in the linked list the page will be, except if it does not have any free slot, in which case it will be at the end. So, the defragmenter tries to have pages full.
51
+
52
+When a page is entirerly free, it is given back to the system immediately. There is no caching of free pages.
53
+
54
+
55
+Parallelism is achieved like this:
56
+
57
+When some threads are waiting for a page to be allocated, free operations are allowed. If a free operation happens before a new page is allocated, any waiting thread will get the slot that is freed on another page.
58
+
59
+Free operations happen in parallel, even for the same page. There is a spin-lock on each page to protect the base pointer of the page's free slots single linked list. But, this is instant. All preparative work happens lockless, then to add the free slot to the page, the page spinlock is acquired, the free slot is prepended to the linked list on the page, the spinlock is released. Such free operations on different pages are totally parallel.
60
+
61
+Once the free operation on a page has finished, the pages double linked list spinlock is acquired to put the page first on that linked list. If the defragmenter is enabled, the spinlock is retained for a little longer, to find the exact position of the page in the linked list.
62
+
63
+During allocations, the reverse order is used. First get the pages double linked list spinlock, get the first page and decrement its free slots counter, then release the spinlock. If the first page does not have any free slots, a page allocation is spawn, without any locks acquired. All threads are spinning waiting for a page with free slots, either from the newly allocated one or from a free operation that may happen in parallel.
64
+
65
+Once a page is acquired, each thread locks its own page to get the first free slot and releases the lock immediately. This is guaranteed to succeed, because when the page was given to that thread its free slots counter was decremented. So, there is a free slot for every thread that got that page. All preparative work to return a pointer to the caller is done lock free. Allocations on different pages are done in parallel, without any intervention between them.
66
+
67
+
68
+## What to expect
69
+
70
+Systems not designed for parallelism achieve their top performance single threaded. The single threaded speed is the baseline. Adding more threads makes them slower.
71
+
72
+The baseline for ARAL is the following, the included stress test when running single threaded:
73
+
74
+```
75
+Running stress test of 1 threads, with 10000 elements each, for 5 seconds...
76
+2023-01-29 17:04:50: netdata INFO : TH[0] : set name of thread 1314983 to TH[0]
77
+ARAL executes 12.27 M malloc and 12.26 M free operations/s
78
+ARAL executes 12.29 M malloc and 12.29 M free operations/s
79
+ARAL executes 12.30 M malloc and 12.30 M free operations/s
80
+ARAL executes 12.30 M malloc and 12.29 M free operations/s
81
+ARAL executes 12.29 M malloc and 12.29 M free operations/s
82
+Waiting the threads to finish...
83
+2023-01-29 17:04:55: netdata INFO : MAIN : ARAL: did 61487356 malloc, 61487356 free, using 1 threads, in 5003808 usecs
84
+```
85
+
86
+The same test with 2 threads, both threads on the same ARAL of course. As you see performance improved:
87
+
88
+```
89
+Running stress test of 2 threads, with 10000 elements each, for 5 seconds...
90
+2023-01-29 17:05:25: netdata INFO : TH[0] : set name of thread 1315537 to TH[0]
91
+2023-01-29 17:05:25: netdata INFO : TH[1] : set name of thread 1315538 to TH[1]
92
+ARAL executes 17.75 M malloc and 17.73 M free operations/s
93
+ARAL executes 17.93 M malloc and 17.93 M free operations/s
94
+ARAL executes 18.17 M malloc and 18.18 M free operations/s
95
+ARAL executes 18.33 M malloc and 18.32 M free operations/s
96
+ARAL executes 18.36 M malloc and 18.36 M free operations/s
97
+Waiting the threads to finish...
98
+2023-01-29 17:05:30: netdata INFO : MAIN : ARAL: did 90976190 malloc, 90976190 free, using 2 threads, in 5029462 usecs
99
+```
100
+
101
+The same test with 4 threads:
102
+
103
+```
104
+Running stress test of 4 threads, with 10000 elements each, for 5 seconds...
105
+2023-01-29 17:10:12: netdata INFO : TH[0] : set name of thread 1319552 to TH[0]
106
+2023-01-29 17:10:12: netdata INFO : TH[1] : set name of thread 1319553 to TH[1]
107
+2023-01-29 17:10:12: netdata INFO : TH[2] : set name of thread 1319554 to TH[2]
108
+2023-01-29 17:10:12: netdata INFO : TH[3] : set name of thread 1319555 to TH[3]
109
+ARAL executes 19.95 M malloc and 19.91 M free operations/s
110
+ARAL executes 20.08 M malloc and 20.08 M free operations/s
111
+ARAL executes 20.85 M malloc and 20.85 M free operations/s
112
+ARAL executes 20.84 M malloc and 20.84 M free operations/s
113
+ARAL executes 21.37 M malloc and 21.37 M free operations/s
114
+Waiting the threads to finish...
115
+2023-01-29 17:10:17: netdata INFO : MAIN : ARAL: did 103549747 malloc, 103549747 free, using 4 threads, in 5023325 usecs
116
+```
117
+
118
+The same with 8 threads:
119
+
120
+```
121
+Running stress test of 8 threads, with 10000 elements each, for 5 seconds...
122
+2023-01-29 17:07:06: netdata INFO : TH[0] : set name of thread 1317608 to TH[0]
123
+2023-01-29 17:07:06: netdata INFO : TH[1] : set name of thread 1317609 to TH[1]
124
+2023-01-29 17:07:06: netdata INFO : TH[2] : set name of thread 1317610 to TH[2]
125
+2023-01-29 17:07:06: netdata INFO : TH[3] : set name of thread 1317611 to TH[3]
126
+2023-01-29 17:07:06: netdata INFO : TH[4] : set name of thread 1317612 to TH[4]
127
+2023-01-29 17:07:06: netdata INFO : TH[5] : set name of thread 1317613 to TH[5]
128
+2023-01-29 17:07:06: netdata INFO : TH[6] : set name of thread 1317614 to TH[6]
129
+2023-01-29 17:07:06: netdata INFO : TH[7] : set name of thread 1317615 to TH[7]
130
+ARAL executes 15.73 M malloc and 15.66 M free operations/s
131
+ARAL executes 13.95 M malloc and 13.94 M free operations/s
132
+ARAL executes 15.59 M malloc and 15.58 M free operations/s
133
+ARAL executes 15.49 M malloc and 15.49 M free operations/s
134
+ARAL executes 16.16 M malloc and 16.16 M free operations/s
135
+Waiting the threads to finish...
136
+2023-01-29 17:07:11: netdata INFO : MAIN : ARAL: did 78427750 malloc, 78427750 free, using 8 threads, in 5088591 usecs
137
+```
138
+
139
+The same with 16 threads:
140
+
141
+```
142
+Running stress test of 16 threads, with 10000 elements each, for 5 seconds...
143
+2023-01-29 17:08:04: netdata INFO : TH[0] : set name of thread 1318663 to TH[0]
144
+2023-01-29 17:08:04: netdata INFO : TH[1] : set name of thread 1318664 to TH[1]
145
+2023-01-29 17:08:04: netdata INFO : TH[2] : set name of thread 1318665 to TH[2]
146
+2023-01-29 17:08:04: netdata INFO : TH[3] : set name of thread 1318666 to TH[3]
147
+2023-01-29 17:08:04: netdata INFO : TH[4] : set name of thread 1318667 to TH[4]
148
+2023-01-29 17:08:04: netdata INFO : TH[5] : set name of thread 1318668 to TH[5]
149
+2023-01-29 17:08:04: netdata INFO : TH[6] : set name of thread 1318669 to TH[6]
150
+2023-01-29 17:08:04: netdata INFO : TH[7] : set name of thread 1318670 to TH[7]
151
+2023-01-29 17:08:04: netdata INFO : TH[8] : set name of thread 1318671 to TH[8]
152
+2023-01-29 17:08:04: netdata INFO : TH[9] : set name of thread 1318672 to TH[9]
153
+2023-01-29 17:08:04: netdata INFO : TH[10] : set name of thread 1318673 to TH[10]
154
+2023-01-29 17:08:04: netdata INFO : TH[11] : set name of thread 1318674 to TH[11]
155
+2023-01-29 17:08:04: netdata INFO : TH[12] : set name of thread 1318675 to TH[12]
156
+2023-01-29 17:08:04: netdata INFO : TH[13] : set name of thread 1318676 to TH[13]
157
+2023-01-29 17:08:04: netdata INFO : TH[14] : set name of thread 1318677 to TH[14]
158
+2023-01-29 17:08:04: netdata INFO : TH[15] : set name of thread 1318678 to TH[15]
159
+ARAL executes 11.77 M malloc and 11.62 M free operations/s
160
+ARAL executes 12.80 M malloc and 12.81 M free operations/s
161
+ARAL executes 13.26 M malloc and 13.25 M free operations/s
162
+ARAL executes 13.30 M malloc and 13.29 M free operations/s
163
+ARAL executes 13.23 M malloc and 13.25 M free operations/s
164
+Waiting the threads to finish...
165
+2023-01-29 17:08:09: netdata INFO : MAIN : ARAL: did 65302122 malloc, 65302122 free, using 16 threads, in 5066009 usecs
166
+```
167
+
168
+As you can see, the top performance is with 4 threads, almost double the single thread speed.
169
+16 threads performance is still better than single threaded, despite the intense concurrency.
libnetdata/aral/aral.c
new
+918
@@ -0,0 +1,918 @@
1
+#include "../libnetdata.h"
2
+#include "aral.h"
3
+
4
+#ifdef NETDATA_TRACE_ALLOCATIONS
5
+#define TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS , const char *file, const char *function, size_t line
6
+#define TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS , file, function, line
7
+#else
8
+#define TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS
9
+#define TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS
10
+#endif
11
+
12
+#define ARAL_FREE_PAGES_DELTA_TO_REARRANGE_LIST 5
13
+
14
+// max file size
15
+#define ARAL_MAX_PAGE_SIZE_MMAP (1*1024*1024*1024)
16
+
17
+// max malloc size
18
+// optimal at current versions of libc is up to 256k
19
+// ideal to have the same overhead as libc is 4k
20
+#define ARAL_MAX_PAGE_SIZE_MALLOC (65*1024)
21
+
22
+typedef struct aral_free {
23
+ size_t size;
24
+ struct aral_free *next;
25
+} ARAL_FREE;
26
+
27
+typedef struct aral_page {
28
+ size_t size; // the allocation size of the page
29
+ const char *filename;
30
+ uint8_t *data;
31
+
32
+ uint32_t free_elements_to_move_first;
33
+ uint32_t max_elements; // the number of elements that can fit on this page
34
+
35
+ struct {
36
+ uint32_t used_elements; // the number of used elements on this page
37
+ uint32_t free_elements; // the number of free elements on this page
38
+ } aral_lock;
39
+
40
+ struct {
41
+ SPINLOCK spinlock;
42
+ ARAL_FREE *list;
43
+ } free;
44
+
45
+ struct aral_page *prev; // the prev page on the list
46
+ struct aral_page *next; // the next page on the list
47
+} ARAL_PAGE;
48
+
49
+struct aral {
50
+ struct {
51
+ char name[ARAL_MAX_NAME + 1];
52
+
53
+ bool lockless;
54
+ bool defragment;
55
+
56
+ size_t element_size; // calculated to take into account ARAL overheads
57
+ size_t max_allocation_size; // calculated in bytes
58
+ size_t page_ptr_offset; // calculated
59
+ size_t natural_page_size; // calculated
60
+
61
+ size_t requested_element_size;
62
+ size_t initial_page_elements;
63
+ size_t max_page_elements;
64
+
65
+ struct {
66
+ bool enabled;
67
+ const char *filename;
68
+ char **cache_dir;
69
+ } mmap;
70
+ } config;
71
+
72
+ struct {
73
+ SPINLOCK spinlock;
74
+ size_t file_number; // for mmap
75
+ struct aral_page *pages; // linked list of pages
76
+
77
+ size_t user_malloc_operations;
78
+ size_t user_free_operations;
79
+ size_t defragment_operations;
80
+ size_t defragment_linked_list_traversals;
81
+ } aral_lock;
82
+
83
+ struct {
84
+ SPINLOCK spinlock;
85
+ size_t allocation_size; // current allocation size
86
+ } adders;
87
+
88
+ struct {
89
+ } atomic;
90
+};
91
+
92
+struct {
93
+ struct {
94
+ struct {
95
+ size_t allocations;
96
+ size_t allocated;
97
+ } structures;
98
+
99
+ struct {
100
+ size_t allocations;
101
+ size_t allocated;
102
+ size_t used;
103
+ } malloc;
104
+
105
+ struct {
106
+ size_t allocations;
107
+ size_t allocated;
108
+ size_t used;
109
+ } mmap;
110
+ } atomic;
111
+} aral_globals = {};
112
+
113
+void aral_get_size_statistics(size_t *structures, size_t *malloc_allocated, size_t *malloc_used, size_t *mmap_allocated, size_t *mmap_used) {
114
+ *structures = __atomic_load_n(&aral_globals.atomic.structures.allocated, __ATOMIC_RELAXED);
115
+ *malloc_allocated = __atomic_load_n(&aral_globals.atomic.malloc.allocated, __ATOMIC_RELAXED);
116
+ *malloc_used = __atomic_load_n(&aral_globals.atomic.malloc.used, __ATOMIC_RELAXED);
117
+ *mmap_allocated = __atomic_load_n(&aral_globals.atomic.mmap.allocated, __ATOMIC_RELAXED);
118
+ *mmap_used = __atomic_load_n(&aral_globals.atomic.mmap.used, __ATOMIC_RELAXED);
119
+}
120
+
121
+#define ARAL_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
122
+static inline size_t natural_alignment(size_t size, size_t alignment) {
123
+ if(unlikely(size % alignment))
124
+ size = size + alignment - (size % alignment);
125
+
126
+ return size;
127
+}
128
+
129
+static size_t aral_align_alloc_size(ARAL *ar, uint64_t size) {
130
+ if(size % ar->config.natural_page_size)
131
+ size += ar->config.natural_page_size - (size % ar->config.natural_page_size) ;
132
+
133
+ if(size % ar->config.element_size)
134
+ size -= size % ar->config.element_size;
135
+
136
+ return size;
137
+}
138
+
139
+static inline void aral_lock(ARAL *ar) {
140
+ if(likely(!ar->config.lockless))
141
+ netdata_spinlock_lock(&ar->aral_lock.spinlock);
142
+}
143
+
144
+static inline void aral_unlock(ARAL *ar) {
145
+ if(likely(!ar->config.lockless))
146
+ netdata_spinlock_unlock(&ar->aral_lock.spinlock);
147
+}
148
+
149
+static void aral_delete_leftover_files(const char *name, const char *path, const char *required_prefix) {
150
+ DIR *dir = opendir(path);
151
+ if(!dir) return;
152
+
153
+ char full_path[FILENAME_MAX + 1];
154
+ size_t len = strlen(required_prefix);
155
+
156
+ struct dirent *de = NULL;
157
+ while((de = readdir(dir))) {
158
+ if(de->d_type == DT_DIR)
159
+ continue;
160
+
161
+ if(strncmp(de->d_name, required_prefix, len) != 0)
162
+ continue;
163
+
164
+ snprintfz(full_path, FILENAME_MAX, "%s/%s", path, de->d_name);
165
+ info("ARAL: '%s' removing left-over file '%s'", name, full_path);
166
+ if(unlikely(unlink(full_path) == -1))
167
+ error("ARAL: '%s' cannot delete file '%s'", name, full_path);
168
+ }
169
+
170
+ closedir(dir);
171
+}
172
+
173
+// ----------------------------------------------------------------------------
174
+// check a free slot
175
+
176
+#ifdef NETDATA_INTERNAL_CHECKS
177
+static inline void aral_free_validate_internal_check(ARAL *ar, ARAL_FREE *fr) {
178
+ if(unlikely(fr->size < ar->config.element_size))
179
+ fatal("ARAL: '%s' free item of size %zu, less than the expected element size %zu",
180
+ ar->config.name, fr->size, ar->config.element_size);
181
+
182
+ if(unlikely(fr->size % ar->config.element_size))
183
+ fatal("ARAL: '%s' free item of size %zu is not multiple to element size %zu",
184
+ ar->config.name, fr->size, ar->config.element_size);
185
+}
186
+#else
187
+#define aral_free_validate_internal_check(ar, fr) debug_dummy()
188
+#endif
189
+
190
+// ----------------------------------------------------------------------------
191
+// find the page a pointer belongs to
192
+
193
+#ifdef NETDATA_INTERNAL_CHECKS
194
+static inline ARAL_PAGE *find_page_with_allocation_internal_check(ARAL *ar, void *ptr) {
195
+ aral_lock(ar);
196
+
197
+ uintptr_t seeking = (uintptr_t)ptr;
198
+ ARAL_PAGE *page;
199
+
200
+ for(page = ar->aral_lock.pages; page ; page = page->next) {
201
+ if(unlikely(seeking >= (uintptr_t)page->data && seeking < (uintptr_t)page->data + page->size))
202
+ break;
203
+ }
204
+
205
+ aral_unlock(ar);
206
+
207
+ return page;
208
+}
209
+#endif
210
+
211
+// ----------------------------------------------------------------------------
212
+// find a page with a free slot (there shouldn't be any)
213
+
214
+#ifdef NETDATA_ARAL_INTERNAL_CHECKS
215
+static inline ARAL_PAGE *find_page_with_free_slots_internal_check___with_aral_lock(ARAL *ar) {
216
+ ARAL_PAGE *page;
217
+
218
+ for(page = ar->aral_lock.pages; page ; page = page->next) {
219
+ if(page->aral_lock.free_elements)
220
+ break;
221
+
222
+ internal_fatal(page->size - page->aral_lock.used_elements * ar->config.element_size >= ar->config.element_size,
223
+ "ARAL: '%s' a page is marked full, but it is not!", ar->config.name);
224
+
225
+ internal_fatal(page->size < page->aral_lock.used_elements * ar->config.element_size,
226
+ "ARAL: '%s' a page has been overflown!", ar->config.name);
227
+ }
228
+
229
+ return page;
230
+}
231
+#endif
232
+
233
+static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
234
+ ARAL_PAGE *page = callocz(1, sizeof(ARAL_PAGE));
235
+ netdata_spinlock_init(&page->free.spinlock);
236
+ page->size = ar->adders.allocation_size;
237
+
238
+ if(page->size > ar->config.max_allocation_size)
239
+ page->size = ar->config.max_allocation_size;
240
+ else
241
+ ar->adders.allocation_size = aral_align_alloc_size(ar, (uint64_t)ar->adders.allocation_size * 4 / 3);
242
+
243
+ page->max_elements = page->aral_lock.free_elements = page->size / ar->config.element_size;
244
+ page->free_elements_to_move_first = page->max_elements / 4;
245
+ if(unlikely(page->free_elements_to_move_first < 1))
246
+ page->free_elements_to_move_first = 1;
247
+
248
+ __atomic_add_fetch(&aral_globals.atomic.structures.allocations, 1, __ATOMIC_RELAXED);
249
+ __atomic_add_fetch(&aral_globals.atomic.structures.allocated, sizeof(ARAL_PAGE), __ATOMIC_RELAXED);
250
+
251
+ if(unlikely(ar->config.mmap.enabled)) {
252
+ ar->aral_lock.file_number++;
253
+ char filename[FILENAME_MAX + 1];
254
+ snprintfz(filename, FILENAME_MAX, "%s/array_alloc.mmap/%s.%zu", *ar->config.mmap.cache_dir, ar->config.mmap.filename, ar->aral_lock.file_number);
255
+ page->filename = strdupz(filename);
256
+ page->data = netdata_mmap(page->filename, page->size, MAP_SHARED, 0, false, NULL);
257
+ if (unlikely(!page->data))
258
+ fatal("ARAL: '%s' cannot allocate aral buffer of size %zu on filename '%s'",
259
+ ar->config.name, page->size, page->filename);
260
+ __atomic_add_fetch(&aral_globals.atomic.mmap.allocations, 1, __ATOMIC_RELAXED);
261
+ __atomic_add_fetch(&aral_globals.atomic.mmap.allocated, page->size, __ATOMIC_RELAXED);
262
+ }
263
+ else {
264
+#ifdef NETDATA_TRACE_ALLOCATIONS
265
+ page->data = mallocz_int(page->size TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
266
+#else
267
+ page->data = mallocz(page->size);
268
+#endif
269
+ __atomic_add_fetch(&aral_globals.atomic.malloc.allocations, 1, __ATOMIC_RELAXED);
270
+ __atomic_add_fetch(&aral_globals.atomic.malloc.allocated, page->size, __ATOMIC_RELAXED);
271
+ }
272
+
273
+ // link the free space to its page
274
+ ARAL_FREE *fr = (ARAL_FREE *)page->data;
275
+ fr->size = page->size;
276
+ fr->next = NULL;
277
+ page->free.list = fr;
278
+
279
+ aral_free_validate_internal_check(ar, fr);
280
+
281
+ return page;
282
+}
283
+
284
+void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
285
+
286
+ // free it
287
+ if (ar->config.mmap.enabled) {
288
+ netdata_munmap(page->data, page->size);
289
+
290
+ if (unlikely(unlink(page->filename) == 1))
291
+ error("Cannot delete file '%s'", page->filename);
292
+
293
+ freez((void *)page->filename);
294
+
295
+ __atomic_sub_fetch(&aral_globals.atomic.mmap.allocations, 1, __ATOMIC_RELAXED);
296
+ __atomic_sub_fetch(&aral_globals.atomic.mmap.allocated, page->size, __ATOMIC_RELAXED);
297
+ }
298
+ else {
299
+#ifdef NETDATA_TRACE_ALLOCATIONS
300
+ freez_int(page->data TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
301
+#else
302
+ freez(page->data);
303
+#endif
304
+ __atomic_sub_fetch(&aral_globals.atomic.malloc.allocations, 1, __ATOMIC_RELAXED);
305
+ __atomic_sub_fetch(&aral_globals.atomic.malloc.allocated, page->size, __ATOMIC_RELAXED);
306
+ }
307
+
308
+ freez(page);
309
+
310
+ __atomic_sub_fetch(&aral_globals.atomic.structures.allocations, 1, __ATOMIC_RELAXED);
311
+ __atomic_sub_fetch(&aral_globals.atomic.structures.allocated, sizeof(ARAL_PAGE), __ATOMIC_RELAXED);
312
+}
313
+
314
+static inline void aral_insert_not_linked_page_with_free_items_to_proper_position___aral_lock_needed(ARAL *ar, ARAL_PAGE *page) {
315
+ ARAL_PAGE *first = ar->aral_lock.pages;
316
+
317
+ if (page->aral_lock.free_elements <= page->free_elements_to_move_first ||
318
+ !first ||
319
+ !first->aral_lock.free_elements ||
320
+ page->aral_lock.free_elements <= first->aral_lock.free_elements + ARAL_FREE_PAGES_DELTA_TO_REARRANGE_LIST) {
321
+ // first position
322
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
323
+ }
324
+ else {
325
+ ARAL_PAGE *second = first->next;
326
+
327
+ if (!second ||
328
+ !second->aral_lock.free_elements ||
329
+ page->aral_lock.free_elements <= second->aral_lock.free_elements)
330
+ // second position
331
+ DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, first, page, prev, next);
332
+ else
333
+ // third position
334
+ DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, second, page, prev, next);
335
+ }
336
+}
337
+
338
+static inline ARAL_PAGE *aral_acquire_a_free_slot(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
339
+ aral_lock(ar);
340
+
341
+ ARAL_PAGE *page = ar->aral_lock.pages;
342
+
343
+ while(!page || !page->aral_lock.free_elements) {
344
+#ifdef NETDATA_ARAL_INTERNAL_CHECKS
345
+ internal_fatal(find_page_with_free_slots_internal_check___with_aral_lock(ar), "ARAL: '%s' found page with free slot!", ar->config.name);
346
+#endif
347
+ aral_unlock(ar);
348
+
349
+ if(netdata_spinlock_trylock(&ar->adders.spinlock)) {
350
+ page = aral_create_page___no_lock_needed(ar TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
351
+
352
+ aral_lock(ar);
353
+ aral_insert_not_linked_page_with_free_items_to_proper_position___aral_lock_needed(ar, page);
354
+ netdata_spinlock_unlock(&ar->adders.spinlock);
355
+ break;
356
+ }
357
+ else {
358
+ aral_lock(ar);
359
+ page = ar->aral_lock.pages;
360
+ }
361
+ }
362
+
363
+ // we have a page
364
+ // and aral locked
365
+
366
+ {
367
+ ARAL_PAGE *first = ar->aral_lock.pages;
368
+ ARAL_PAGE *second = first->next;
369
+
370
+ if (!second ||
371
+ !second->aral_lock.free_elements ||
372
+ first->aral_lock.free_elements <= second->aral_lock.free_elements + ARAL_FREE_PAGES_DELTA_TO_REARRANGE_LIST)
373
+ page = first;
374
+ else {
375
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, second, prev, next);
376
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, second, prev, next);
377
+ page = second;
378
+ }
379
+ }
380
+
381
+ internal_fatal(!page || !page->aral_lock.free_elements,
382
+ "ARAL: '%s' selected page does not have a free slot in it",
383
+ ar->config.name);
384
+
385
+ internal_fatal(page->max_elements != page->aral_lock.used_elements + page->aral_lock.free_elements,
386
+ "ARAL: '%s' page element counters do not match, "
387
+ "page says it can handle %zu elements, "
388
+ "but there are %zu used and %zu free items, "
389
+ "total %zu items",
390
+ ar->config.name,
391
+ (size_t)page->max_elements,
392
+ (size_t)page->aral_lock.used_elements, (size_t)page->aral_lock.free_elements,
393
+ (size_t)page->aral_lock.used_elements + (size_t)page->aral_lock.free_elements
394
+ );
395
+
396
+ ar->aral_lock.user_malloc_operations++;
397
+
398
+ // acquire a slot for the caller
399
+ page->aral_lock.used_elements++;
400
+ if(--page->aral_lock.free_elements == 0) {
401
+ // we are done with this page
402
+ // move the full page last
403
+ // so that pages with free items remain first in the list
404
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
405
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
406
+ }
407
+
408
+ aral_unlock(ar);
409
+
410
+ return page;
411
+}
412
+
413
+void *aral_mallocz_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
414
+
415
+ ARAL_PAGE *page = aral_acquire_a_free_slot(ar TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
416
+
417
+ netdata_spinlock_lock(&page->free.spinlock);
418
+
419
+ internal_fatal(!page->free.list,
420
+ "ARAL: '%s' free item to use, cannot be NULL.", ar->config.name);
421
+
422
+ internal_fatal(page->free.list->size < ar->config.element_size,
423
+ "ARAL: '%s' free item size %zu, cannot be smaller than %zu",
424
+ ar->config.name, page->free.list->size, ar->config.element_size);
425
+
426
+ ARAL_FREE *found_fr = page->free.list;
427
+
428
+ // check if the remaining size (after we use this slot) is not enough for another element
429
+ if(unlikely(found_fr->size - ar->config.element_size < ar->config.element_size)) {
430
+ // we can use the entire free space entry
431
+
432
+ page->free.list = found_fr->next;
433
+ }
434
+ else {
435
+ // we can split the free space entry
436
+
437
+ uint8_t *data = (uint8_t *)found_fr;
438
+ ARAL_FREE *fr = (ARAL_FREE *)&data[ar->config.element_size];
439
+ fr->size = found_fr->size - ar->config.element_size;
440
+
441
+ // link the free slot first in the page
442
+ fr->next = found_fr->next;
443
+ page->free.list = fr;
444
+
445
+ aral_free_validate_internal_check(ar, fr);
446
+ }
447
+
448
+ netdata_spinlock_unlock(&page->free.spinlock);
449
+
450
+ // put the page pointer after the element
451
+ uint8_t *data = (uint8_t *)found_fr;
452
+ ARAL_PAGE **page_ptr = (ARAL_PAGE **)&data[ar->config.page_ptr_offset];
453
+ *page_ptr = page;
454
+
455
+ if(unlikely(ar->config.mmap.enabled))
456
+ __atomic_add_fetch(&aral_globals.atomic.mmap.used, ar->config.element_size, __ATOMIC_RELAXED);
457
+ else
458
+ __atomic_add_fetch(&aral_globals.atomic.malloc.used, ar->config.element_size, __ATOMIC_RELAXED);
459
+
460
+ return (void *)found_fr;
461
+}
462
+
463
+static inline ARAL_PAGE *aral_ptr_to_page___must_NOT_have_aral_lock(ARAL *ar, void *ptr) {
464
+ // given a data pointer we returned before,
465
+ // find the ARAL_PAGE it belongs to
466
+
467
+ uint8_t *data = (uint8_t *)ptr;
468
+ ARAL_PAGE **page_ptr = (ARAL_PAGE **)&data[ar->config.page_ptr_offset];
469
+ ARAL_PAGE *page = *page_ptr;
470
+
471
+#ifdef NETDATA_INTERNAL_CHECKS
472
+ // make it NULL so that we will fail on double free
473
+ // do not enable this on production, because the MMAP file
474
+ // will need to be saved again!
475
+ *page_ptr = NULL;
476
+#endif
477
+
478
+#ifdef NETDATA_ARAL_INTERNAL_CHECKS
479
+ {
480
+ // find the page ptr belongs
481
+ ARAL_PAGE *page2 = find_page_with_allocation_internal_check(ar, ptr);
482
+
483
+ internal_fatal(page != page2,
484
+ "ARAL: '%s' page pointers do not match!",
485
+ ar->name);
486
+
487
+ internal_fatal(!page2,
488
+ "ARAL: '%s' free of pointer %p is not in ARAL address space.",
489
+ ar->name, ptr);
490
+ }
491
+#endif
492
+
493
+ internal_fatal(!page,
494
+ "ARAL: '%s' possible corruption or double free of pointer %p",
495
+ ar->config.name, ptr);
496
+
497
+ return page;
498
+}
499
+
500
+static void aral_defrag_sorted_page_position___aral_lock_needed(ARAL *ar, ARAL_PAGE *page) {
501
+ ARAL_PAGE *tmp;
502
+
503
+ int action = 0; (void)action;
504
+ size_t move_later = 0, move_earlier = 0;
505
+
506
+ for(tmp = page->next ;
507
+ tmp && tmp->aral_lock.free_elements && tmp->aral_lock.free_elements < page->aral_lock.free_elements ;
508
+ tmp = tmp->next)
509
+ move_later++;
510
+
511
+ if(!tmp && page->next) {
512
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
513
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
514
+ action = 1;
515
+ }
516
+ else if(tmp != page->next) {
517
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
518
+ DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(ar->aral_lock.pages, tmp, page, prev, next);
519
+ action = 2;
520
+ }
521
+ else {
522
+ for(tmp = (page == ar->aral_lock.pages) ? NULL : page->prev ;
523
+ tmp && (!tmp->aral_lock.free_elements || tmp->aral_lock.free_elements > page->aral_lock.free_elements);
524
+ tmp = (tmp == ar->aral_lock.pages) ? NULL : tmp->prev)
525
+ move_earlier++;
526
+
527
+ if(!tmp) {
528
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
529
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
530
+ action = 3;
531
+ }
532
+ else if(tmp != page->prev){
533
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
534
+ DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, tmp, page, prev, next);
535
+ action = 4;
536
+ }
537
+ }
538
+
539
+ ar->aral_lock.defragment_operations++;
540
+ ar->aral_lock.defragment_linked_list_traversals += move_earlier + move_later;
541
+
542
+ internal_fatal(page->next && page->next->aral_lock.free_elements && page->next->aral_lock.free_elements < page->aral_lock.free_elements,
543
+ "ARAL: '%s' item should be later in the list", ar->config.name);
544
+
545
+ internal_fatal(page != ar->aral_lock.pages && (!page->prev->aral_lock.free_elements || page->prev->aral_lock.free_elements > page->aral_lock.free_elements),
546
+ "ARAL: '%s' item should be earlier in the list", ar->config.name);
547
+}
548
+
549
+static inline void aral_move_page_with_free_list___aral_lock_needed(ARAL *ar, ARAL_PAGE *page) {
550
+ if(unlikely(page == ar->aral_lock.pages))
551
+ // we are the first already
552
+ return;
553
+
554
+ if(likely(!ar->config.defragment)) {
555
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
556
+ aral_insert_not_linked_page_with_free_items_to_proper_position___aral_lock_needed(ar, page);
557
+ }
558
+ else
559
+ aral_defrag_sorted_page_position___aral_lock_needed(ar, page);
560
+}
561
+
562
+void aral_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
563
+ if(unlikely(!ptr)) return;
564
+
565
+ // get the page pointer
566
+ ARAL_PAGE *page = aral_ptr_to_page___must_NOT_have_aral_lock(ar, ptr);
567
+
568
+ if(unlikely(ar->config.mmap.enabled))
569
+ __atomic_sub_fetch(&aral_globals.atomic.mmap.used, ar->config.element_size, __ATOMIC_RELAXED);
570
+ else
571
+ __atomic_sub_fetch(&aral_globals.atomic.malloc.used, ar->config.element_size, __ATOMIC_RELAXED);
572
+
573
+ // make this element available
574
+ ARAL_FREE *fr = (ARAL_FREE *)ptr;
575
+ fr->size = ar->config.element_size;
576
+
577
+ netdata_spinlock_lock(&page->free.spinlock);
578
+ fr->next = page->free.list;
579
+ page->free.list = fr;
580
+ netdata_spinlock_unlock(&page->free.spinlock);
581
+
582
+ aral_lock(ar);
583
+
584
+ internal_fatal(!page->aral_lock.used_elements,
585
+ "ARAL: '%s' pointer %p is inside a page without any active allocations.",
586
+ ar->config.name, ptr);
587
+
588
+ internal_fatal(page->max_elements != page->aral_lock.used_elements + page->aral_lock.free_elements,
589
+ "ARAL: '%s' page element counters do not match, "
590
+ "page says it can handle %zu elements, "
591
+ "but there are %zu used and %zu free items, "
592
+ "total %zu items",
593
+ ar->config.name,
594
+ (size_t)page->max_elements,
595
+ (size_t)page->aral_lock.used_elements, (size_t)page->aral_lock.free_elements,
596
+ (size_t)page->aral_lock.used_elements + (size_t)page->aral_lock.free_elements
597
+ );
598
+
599
+ page->aral_lock.used_elements--;
600
+ page->aral_lock.free_elements++;
601
+
602
+ ar->aral_lock.user_free_operations++;
603
+
604
+ // if the page is empty, release it
605
+ if(unlikely(!page->aral_lock.used_elements)) {
606
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
607
+ aral_unlock(ar);
608
+ aral_del_page___no_lock_needed(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
609
+ }
610
+ else {
611
+ aral_move_page_with_free_list___aral_lock_needed(ar, page);
612
+ aral_unlock(ar);
613
+ }
614
+}
615
+
616
+void aral_destroy_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
617
+ aral_lock(ar);
618
+
619
+ ARAL_PAGE *page;
620
+ while((page = ar->aral_lock.pages)) {
621
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
622
+ aral_del_page___no_lock_needed(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
623
+ }
624
+
625
+ aral_unlock(ar);
626
+ freez(ar);
627
+}
628
+
629
+ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_elements, const char *filename, char **cache_dir, bool mmap, bool lockless) {
630
+ ARAL *ar = callocz(1, sizeof(ARAL));
631
+ ar->config.requested_element_size = element_size;
632
+ ar->config.initial_page_elements = initial_page_elements;
633
+ ar->config.max_page_elements = max_page_elements;
634
+ ar->config.mmap.filename = filename;
635
+ ar->config.mmap.cache_dir = cache_dir;
636
+ ar->config.mmap.enabled = mmap;
637
+ ar->config.lockless = lockless;
638
+ ar->config.defragment = false;
639
+ strncpyz(ar->config.name, name, ARAL_MAX_NAME);
640
+ netdata_spinlock_init(&ar->aral_lock.spinlock);
641
+
642
+ long int page_size = sysconf(_SC_PAGE_SIZE);
643
+ if (unlikely(page_size == -1))
644
+ ar->config.natural_page_size = 4096;
645
+ else
646
+ ar->config.natural_page_size = page_size;
647
+
648
+ // we need to add a page pointer after the element
649
+ // so, first align the element size to the pointer size
650
+ ar->config.element_size = natural_alignment(ar->config.requested_element_size, sizeof(uintptr_t));
651
+
652
+ // then add the size of a pointer to it
653
+ ar->config.element_size += sizeof(uintptr_t);
654
+
655
+ // make sure it is at least what we need for an ARAL_FREE slot
656
+ if (ar->config.element_size < sizeof(ARAL_FREE))
657
+ ar->config.element_size = sizeof(ARAL_FREE);
658
+
659
+ // and finally align it to the natural alignment
660
+ ar->config.element_size = natural_alignment(ar->config.element_size, ARAL_NATURAL_ALIGNMENT);
661
+
662
+ // we write the page pointer just after each element
663
+ ar->config.page_ptr_offset = ar->config.element_size - sizeof(uintptr_t);
664
+
665
+ if(ar->config.requested_element_size + sizeof(uintptr_t) > ar->config.element_size)
666
+ fatal("ARAL: '%s' failed to calculate properly page_ptr_offset: "
667
+ "element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, "
668
+ "final element size %zu, page_ptr_offset %zu",
669
+ ar->config.name, ar->config.requested_element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT,
670
+ ar->config.element_size, ar->config.page_ptr_offset);
671
+
672
+ //info("ARAL: element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, final element size %zu, page_ptr_offset %zu",
673
+ // ar->element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT, ar->internal.element_size, ar->internal.page_ptr_offset);
674
+
675
+
676
+ if (ar->config.initial_page_elements < 2)
677
+ ar->config.initial_page_elements = 2;
678
+
679
+ if(ar->config.mmap.enabled && (!ar->config.mmap.cache_dir || !*ar->config.mmap.cache_dir)) {
680
+ error("ARAL: '%s' mmap cache directory is not configured properly, disabling mmap.", ar->config.name);
681
+ ar->config.mmap.enabled = false;
682
+ internal_fatal(true, "ARAL: '%s' mmap cache directory is not configured properly", ar->config.name);
683
+ }
684
+
685
+ uint64_t max_alloc_size;
686
+ if(!ar->config.max_page_elements)
687
+ max_alloc_size = ar->config.mmap.enabled ? ARAL_MAX_PAGE_SIZE_MMAP : ARAL_MAX_PAGE_SIZE_MALLOC;
688
+ else
689
+ max_alloc_size = ar->config.max_page_elements * ar->config.element_size;
690
+
691
+ ar->config.max_allocation_size = aral_align_alloc_size(ar, max_alloc_size);
692
+ ar->adders.allocation_size = aral_align_alloc_size(ar, (uint64_t)ar->config.element_size * ar->config.initial_page_elements);
693
+ ar->aral_lock.pages = NULL;
694
+ ar->aral_lock.file_number = 0;
695
+
696
+ if(ar->config.mmap.enabled) {
697
+ char directory_name[FILENAME_MAX + 1];
698
+ snprintfz(directory_name, FILENAME_MAX, "%s/array_alloc.mmap", *ar->config.mmap.cache_dir);
699
+ int r = mkdir(directory_name, 0775);
700
+ if (r != 0 && errno != EEXIST)
701
+ fatal("Cannot create directory '%s'", directory_name);
702
+
703
+ char file[FILENAME_MAX + 1];
704
+ snprintfz(file, FILENAME_MAX, "%s.", ar->config.mmap.filename);
705
+ aral_delete_leftover_files(ar->config.name, directory_name, file);
706
+ }
707
+
708
+ internal_error(true,
709
+ "ARAL: '%s' "
710
+ "element size %zu (requested %zu bytes), "
711
+ "min elements per page %zu (requested %zu), "
712
+ "max elements per page %zu (requested %zu), "
713
+ "max page size %zu bytes, "
714
+ , ar->config.name
715
+ , ar->config.element_size, ar->config.requested_element_size
716
+ , ar->adders.allocation_size / ar->config.element_size, ar->config.initial_page_elements
717
+ , ar->config.max_allocation_size / ar->config.element_size, ar->config.max_page_elements
718
+ , ar->config.max_allocation_size
719
+ );
720
+
721
+ __atomic_add_fetch(&aral_globals.atomic.structures.allocations, 1, __ATOMIC_RELAXED);
722
+ __atomic_add_fetch(&aral_globals.atomic.structures.allocated, sizeof(ARAL), __ATOMIC_RELAXED);
723
+ return ar;
724
+}
725
+
726
+// ----------------------------------------------------------------------------
727
+// unittest
728
+
729
+struct aral_unittest_config {
730
+ bool single_threaded;
731
+ bool stop;
732
+ ARAL *ar;
733
+ size_t elements;
734
+ size_t threads;
735
+ int errors;
736
+};
737
+
738
+static void *aral_test_thread(void *ptr) {
739
+ struct aral_unittest_config *auc = ptr;
740
+ ARAL *ar = auc->ar;
741
+ size_t elements = auc->elements;
742
+
743
+ void **pointers = callocz(elements, sizeof(void *));
744
+
745
+ do {
746
+ for (size_t i = 0; i < elements; i++) {
747
+ pointers[i] = aral_mallocz(ar);
748
+ }
749
+
750
+ for (size_t div = 5; div >= 2; div--) {
751
+ for (size_t i = 0; i < elements / div; i++) {
752
+ aral_freez(ar, pointers[i]);
753
+ pointers[i] = NULL;
754
+ }
755
+
756
+ for (size_t i = 0; i < elements / div; i++) {
757
+ pointers[i] = aral_mallocz(ar);
758
+ }
759
+ }
760
+
761
+ for (size_t step = 50; step >= 10; step -= 10) {
762
+ for (size_t i = 0; i < elements; i += step) {
763
+ aral_freez(ar, pointers[i]);
764
+ pointers[i] = NULL;
765
+ }
766
+
767
+ for (size_t i = 0; i < elements; i += step) {
768
+ pointers[i] = aral_mallocz(ar);
769
+ }
770
+ }
771
+
772
+ for (size_t i = 0; i < elements; i++) {
773
+ aral_freez(ar, pointers[i]);
774
+ pointers[i] = NULL;
775
+ }
776
+
777
+ if (auc->single_threaded && ar->aral_lock.pages) {
778
+ fprintf(stderr, "\n\nARAL leftovers detected (1)\n\n");
779
+ __atomic_add_fetch(&auc->errors, 1, __ATOMIC_RELAXED);
780
+ }
781
+
782
+ if(!auc->single_threaded && __atomic_load_n(&auc->stop, __ATOMIC_RELAXED))
783
+ break;
784
+
785
+ for (size_t i = 0; i < elements; i++) {
786
+ pointers[i] = aral_mallocz(ar);
787
+ }
788
+
789
+ size_t increment = elements / ar->config.max_page_elements;
790
+ for (size_t all = increment; all <= elements / 2; all += increment) {
791
+
792
+ size_t to_free = all % ar->config.max_page_elements;
793
+ size_t step = elements / to_free;
794
+ if(!step) step = 1;
795
+
796
+ // fprintf(stderr, "all %zu, to free %zu, step %zu\n", all, to_free, step);
797
+
798
+ size_t free_list[to_free];
799
+ for (size_t i = 0; i < to_free; i++) {
800
+ size_t pos = step * i;
801
+ aral_freez(ar, pointers[pos]);
802
+ pointers[pos] = NULL;
803
+ free_list[i] = pos;
804
+ }
805
+
806
+ for (size_t i = 0; i < to_free; i++) {
807
+ size_t pos = free_list[i];
808
+ pointers[pos] = aral_mallocz(ar);
809
+ }
810
+ }
811
+
812
+ for (size_t i = 0; i < elements; i++) {
813
+ aral_freez(ar, pointers[i]);
814
+ pointers[i] = NULL;
815
+ }
816
+
817
+ if (auc->single_threaded && ar->aral_lock.pages) {
818
+ fprintf(stderr, "\n\nARAL leftovers detected (2)\n\n");
819
+ __atomic_add_fetch(&auc->errors, 1, __ATOMIC_RELAXED);
820
+ }
821
+
822
+ } while(!auc->single_threaded && !__atomic_load_n(&auc->stop, __ATOMIC_RELAXED));
823
+
824
+ freez(pointers);
825
+
826
+ return ptr;
827
+}
828
+
829
+int aral_stress_test(size_t threads, size_t elements, size_t seconds) {
830
+ fprintf(stderr, "Running stress test of %zu threads, with %zu elements each, for %zu seconds...\n",
831
+ threads, elements, seconds);
832
+
833
+ memset(&aral_globals, 0, sizeof(aral_globals));
834
+
835
+ struct aral_unittest_config auc = {
836
+ .single_threaded = false,
837
+ .threads = threads,
838
+ .ar = aral_create("aral-test", 20, 10, 1024, "test-aral", NULL, false, false),
839
+ .elements = elements,
840
+ .errors = 0,
841
+ };
842
+
843
+ usec_t started_ut = now_monotonic_usec();
844
+ netdata_thread_t thread_ptrs[threads];
845
+
846
+ for(size_t i = 0; i < threads ; i++) {
847
+ char tag[NETDATA_THREAD_NAME_MAX + 1];
848
+ snprintfz(tag, NETDATA_THREAD_NAME_MAX, "TH[%zu]", i);
849
+ netdata_thread_create(&thread_ptrs[i], tag,
850
+ NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG,
851
+ aral_test_thread, &auc);
852
+ }
853
+
854
+ size_t malloc_done = 0;
855
+ size_t free_done = 0;
856
+ size_t countdown = seconds;
857
+ while(countdown-- > 0) {
858
+ sleep_usec(1 * USEC_PER_SEC);
859
+ aral_lock(auc.ar);
860
+ size_t m = auc.ar->aral_lock.user_malloc_operations;
861
+ size_t f = auc.ar->aral_lock.user_free_operations;
862
+ aral_unlock(auc.ar);
863
+ fprintf(stderr, "ARAL executes %0.2f M malloc and %0.2f M free operations/s\n",
864
+ (double)(m - malloc_done) / 1000000.0, (double)(f - free_done) / 1000000.0);
865
+ malloc_done = m;
866
+ free_done = f;
867
+ }
868
+
869
+ __atomic_store_n(&auc.stop, true, __ATOMIC_RELAXED);
870
+
871
+// fprintf(stderr, "Cancelling the threads...\n");
872
+// for(size_t i = 0; i < threads ; i++) {
873
+// netdata_thread_cancel(thread_ptrs[i]);
874
+// }
875
+
876
+ fprintf(stderr, "Waiting the threads to finish...\n");
877
+ for(size_t i = 0; i < threads ; i++) {
878
+ netdata_thread_join(thread_ptrs[i], NULL);
879
+ }
880
+
881
+ usec_t ended_ut = now_monotonic_usec();
882
+
883
+ if (auc.ar->aral_lock.pages) {
884
+ fprintf(stderr, "\n\nARAL leftovers detected (3)\n\n");
885
+ __atomic_add_fetch(&auc.errors, 1, __ATOMIC_RELAXED);
886
+ }
887
+
888
+ info("ARAL: did %zu malloc, %zu free, "
889
+ "using %zu threads, in %llu usecs",
890
+ auc.ar->aral_lock.user_malloc_operations,
891
+ auc.ar->aral_lock.user_free_operations,
892
+ threads,
893
+ ended_ut - started_ut);
894
+
895
+ aral_destroy(auc.ar);
896
+
897
+ return auc.errors;
898
+}
899
+
900
+int aral_unittest(size_t elements) {
901
+ char *cache_dir = "/tmp/";
902
+
903
+ struct aral_unittest_config auc = {
904
+ .single_threaded = true,
905
+ .threads = 1,
906
+ .ar = aral_create("aral-test", 20, 10, 1024, "test-aral", &cache_dir, false, false),
907
+ .elements = elements,
908
+ .errors = 0,
909
+ };
910
+
911
+ aral_test_thread(&auc);
912
+
913
+ aral_destroy(auc.ar);
914
+
915
+ int errors = aral_stress_test(2, elements, 5);
916
+
917
+ return auc.errors + errors;
918
+}
libnetdata/aral/aral.h
new
+37
@@ -0,0 +1,37 @@
1
+
2
+#ifndef ARAL_H
3
+#define ARAL_H 1
4
+
5
+#include "../libnetdata.h"
6
+
7
+#define ARAL_MAX_NAME 23
8
+
9
+typedef struct aral ARAL;
10
+
11
+ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_elements, const char *filename, char **cache_dir, bool mmap, bool lockless);
12
+int aral_unittest(size_t elements);
13
+void aral_get_size_statistics(size_t *structures, size_t *malloc_allocated, size_t *malloc_used, size_t *mmap_allocated, size_t *mmap_used);
14
+
15
+#ifdef NETDATA_TRACE_ALLOCATIONS
16
+
17
+#define aral_mallocz(ar) aral_mallocz_internal(ar, __FILE__, __FUNCTION__, __LINE__)
18
+#define aral_freez(ar, ptr) aral_freez_internal(ar, ptr, __FILE__, __FUNCTION__, __LINE__)
19
+#define aral_destroy(ar) aral_destroy_internal(ar, __FILE__, __FUNCTION__, __LINE__)
20
+
21
+void *aral_mallocz_internal(ARAL *ar, const char *file, const char *function, size_t line);
22
+void aral_freez_internal(ARAL *ar, void *ptr, const char *file, const char *function, size_t line);
23
+void aral_destroy_internal(ARAL *ar, const char *file, const char *function, size_t line);
24
+
25
+#else // NETDATA_TRACE_ALLOCATIONS
26
+
27
+#define aral_mallocz(ar) aral_mallocz_internal(ar)
28
+#define aral_freez(ar, ptr) aral_freez_internal(ar, ptr)
29
+#define aral_destroy(ar) aral_destroy_internal(ar)
30
+
31
+void *aral_mallocz_internal(ARAL *ar);
32
+void aral_freez_internal(ARAL *ar, void *ptr);
33
+void aral_destroy_internal(ARAL *ar);
34
+
35
+#endif // NETDATA_TRACE_ALLOCATIONS
36
+
37
+#endif // ARAL_H
libnetdata/arrayalloc/README.md
deleted
-7
@@ -1,7 +0,0 @@
1
-<!--
2
-title: "Array Allocator"
3
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/arrayalloc/README.md
4
--->
5
-
6
-# Array Allocator
7
-
libnetdata/arrayalloc/arrayalloc.c
deleted
-501
@@ -1,501 +0,0 @@
1
-#include "../libnetdata.h"
2
-#include "arrayalloc.h"
3
-
4
-#ifdef NETDATA_TRACE_ALLOCATIONS
5
-#define TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS , const char *file, const char *function, size_t line
6
-#define TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS , file, function, line
7
-#else
8
-#define TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS
9
-#define TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS
10
-#endif
11
-
12
-// max file size
13
-#define ARAL_MAX_PAGE_SIZE_MMAP (1*1024*1024*1024)
14
-
15
-// max malloc size
16
-// optimal at current versions of libc is up to 256k
17
-// ideal to have the same overhead as libc is 4k
18
-#define ARAL_MAX_PAGE_SIZE_MALLOC (64*1024)
19
-
20
-typedef struct arrayalloc_free {
21
- size_t size;
22
- struct arrayalloc_page *page;
23
- struct arrayalloc_free *next;
24
-} ARAL_FREE;
25
-
26
-typedef struct arrayalloc_page {
27
- const char *filename;
28
- size_t size; // the total size of the page
29
- size_t used_elements; // the total number of used elements on this page
30
- uint8_t *data;
31
- ARAL_FREE *free_list;
32
- struct arrayalloc_page *prev; // the prev page on the list
33
- struct arrayalloc_page *next; // the next page on the list
34
-} ARAL_PAGE;
35
-
36
-#define ARAL_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
37
-static inline size_t natural_alignment(size_t size, size_t alignment) {
38
- if(unlikely(size % alignment))
39
- size = size + alignment - (size % alignment);
40
-
41
- return size;
42
-}
43
-
44
-static void arrayalloc_delete_leftover_files(const char *path, const char *required_prefix) {
45
- DIR *dir = opendir(path);
46
- if(!dir) return;
47
-
48
- char full_path[FILENAME_MAX + 1];
49
- size_t len = strlen(required_prefix);
50
-
51
- struct dirent *de = NULL;
52
- while((de = readdir(dir))) {
53
- if(de->d_type == DT_DIR)
54
- continue;
55
-
56
- if(strncmp(de->d_name, required_prefix, len) != 0)
57
- continue;
58
-
59
- snprintfz(full_path, FILENAME_MAX, "%s/%s", path, de->d_name);
60
- info("ARRAYALLOC: removing left-over file '%s'", full_path);
61
- if(unlikely(unlink(full_path) == -1))
62
- error("Cannot delete file '%s'", full_path);
63
- }
64
-
65
- closedir(dir);
66
-}
67
-
68
-// ----------------------------------------------------------------------------
69
-// arrayalloc_init()
70
-
71
-static void arrayalloc_init(ARAL *ar) {
72
- static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
73
- netdata_mutex_lock(&mutex);
74
-
75
- if(!ar->internal.initialized) {
76
- netdata_spinlock_init(&ar->internal.spinlock);
77
-
78
- long int page_size = sysconf(_SC_PAGE_SIZE);
79
- if (unlikely(page_size == -1))
80
- ar->internal.natural_page_size = 4096;
81
- else
82
- ar->internal.natural_page_size = page_size;
83
-
84
- // we need to add a page pointer after the element
85
- // so, first align the element size to the pointer size
86
- ar->internal.element_size = natural_alignment(ar->requested_element_size, sizeof(uintptr_t));
87
-
88
- // then add the size of a pointer to it
89
- ar->internal.element_size += sizeof(uintptr_t);
90
-
91
- // make sure it is at least what we need for an ARAL_FREE slot
92
- if (ar->internal.element_size < sizeof(ARAL_FREE))
93
- ar->internal.element_size = sizeof(ARAL_FREE);
94
-
95
- // and finally align it to the natural alignment
96
- ar->internal.element_size = natural_alignment(ar->internal.element_size, ARAL_NATURAL_ALIGNMENT);
97
-
98
- // we write the page pointer just after each element
99
- ar->internal.page_ptr_offset = ar->internal.element_size - sizeof(uintptr_t);
100
-
101
- if(ar->requested_element_size + sizeof(uintptr_t) > ar->internal.element_size)
102
- fatal("ARRAYALLOC: failed to calculate properly page_ptr_offset: element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, final element size %zu, page_ptr_offset %zu",
103
- ar->requested_element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT, ar->internal.element_size, ar->internal.page_ptr_offset);
104
-
105
- //info("ARRAYALLOC: element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, final element size %zu, page_ptr_offset %zu",
106
- // ar->element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT, ar->internal.element_size, ar->internal.page_ptr_offset);
107
-
108
- if (ar->initial_elements < 10)
109
- ar->initial_elements = 10;
110
-
111
- ar->internal.mmap = (ar->use_mmap && ar->cache_dir && *ar->cache_dir) ? true : false;
112
- ar->internal.max_alloc_size = ar->internal.mmap ? ARAL_MAX_PAGE_SIZE_MMAP : ARAL_MAX_PAGE_SIZE_MALLOC;
113
-
114
- if(ar->internal.max_alloc_size % ar->internal.natural_page_size)
115
- ar->internal.max_alloc_size += ar->internal.natural_page_size - (ar->internal.max_alloc_size % ar->internal.natural_page_size) ;
116
-
117
- if(ar->internal.max_alloc_size % ar->internal.element_size)
118
- ar->internal.max_alloc_size -= ar->internal.max_alloc_size % ar->internal.element_size;
119
-
120
- ar->internal.pages = NULL;
121
- ar->internal.allocation_multiplier = 1;
122
- ar->internal.file_number = 0;
123
-
124
- if(ar->internal.mmap) {
125
- char directory_name[FILENAME_MAX + 1];
126
- snprintfz(directory_name, FILENAME_MAX, "%s/array_alloc.mmap", *ar->cache_dir);
127
- int r = mkdir(directory_name, 0775);
128
- if (r != 0 && errno != EEXIST)
129
- fatal("Cannot create directory '%s'", directory_name);
130
-
131
- char filename[FILENAME_MAX + 1];
132
- snprintfz(filename, FILENAME_MAX, "%s.", ar->filename);
133
- arrayalloc_delete_leftover_files(directory_name, filename);
134
- }
135
-
136
- ar->internal.initialized = true;
137
- }
138
-
139
- netdata_mutex_unlock(&mutex);
140
-}
141
-
142
-// ----------------------------------------------------------------------------
143
-// check a free slot
144
-
145
-#ifdef NETDATA_INTERNAL_CHECKS
146
-static inline void arrayalloc_free_validate_internal_check(ARAL *ar, ARAL_FREE *fr) {
147
- if(unlikely(fr->size < ar->internal.element_size))
148
- fatal("ARRAYALLOC: free item of size %zu, less than the expected element size %zu", fr->size, ar->internal.element_size);
149
-
150
- if(unlikely(fr->size % ar->internal.element_size))
151
- fatal("ARRAYALLOC: free item of size %zu is not multiple to element size %zu", fr->size, ar->internal.element_size);
152
-}
153
-#else
154
-#define arrayalloc_free_validate_internal_check(ar, fr) debug_dummy()
155
-#endif
156
-
157
-// ----------------------------------------------------------------------------
158
-// find the page a pointer belongs to
159
-
160
-#ifdef NETDATA_INTERNAL_CHECKS
161
-static inline ARAL_PAGE *find_page_with_allocation_internal_check(ARAL *ar, void *ptr) {
162
- uintptr_t seeking = (uintptr_t)ptr;
163
- ARAL_PAGE *page;
164
-
165
- for(page = ar->internal.pages; page ; page = page->next) {
166
- if(unlikely(seeking >= (uintptr_t)page->data && seeking < (uintptr_t)page->data + page->size))
167
- break;
168
- }
169
-
170
- return page;
171
-}
172
-#endif
173
-
174
-// ----------------------------------------------------------------------------
175
-// find a page with a free slot (there shouldn't be any)
176
-
177
-#ifdef NETDATA_ARRAYALLOC_INTERNAL_CHECKS
178
-static inline ARAL_PAGE *find_page_with_free_slots_internal_check(ARAL *ar) {
179
- ARAL_PAGE *page;
180
-
181
- for(page = ar->internal.pages; page ; page = page->next) {
182
- if(page->free_list)
183
- break;
184
-
185
- internal_fatal(page->size - page->used_elements * ar->internal.element_size >= ar->internal.element_size,
186
- "ARRAYALLOC: a page is marked full, but it is not!");
187
-
188
- internal_fatal(page->size < page->used_elements * ar->internal.element_size,
189
- "ARRAYALLOC: a page has been overflown!");
190
- }
191
-
192
- return page;
193
-}
194
-#endif
195
-
196
-static void arrayalloc_add_page(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
197
- if(unlikely(!ar->internal.initialized))
198
- arrayalloc_init(ar);
199
-
200
- ARAL_PAGE *page = callocz(1, sizeof(ARAL_PAGE));
201
- page->size = ar->initial_elements * ar->internal.element_size * ar->internal.allocation_multiplier;
202
- if(page->size > ar->internal.max_alloc_size)
203
- page->size = ar->internal.max_alloc_size;
204
- else
205
- ar->internal.allocation_multiplier *= 2;
206
-
207
- if(ar->internal.mmap) {
208
- ar->internal.file_number++;
209
- char filename[FILENAME_MAX + 1];
210
- snprintfz(filename, FILENAME_MAX, "%s/array_alloc.mmap/%s.%zu", *ar->cache_dir, ar->filename, ar->internal.file_number);
211
- page->filename = strdupz(filename);
212
- page->data = netdata_mmap(page->filename, page->size, MAP_SHARED, 0, false, NULL);
213
- if (unlikely(!page->data))
214
- fatal("Cannot allocate arrayalloc buffer of size %zu on filename '%s'", page->size, page->filename);
215
- }
216
- else {
217
-#ifdef NETDATA_TRACE_ALLOCATIONS
218
- page->data = mallocz_int(page->size TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
219
-#else
220
- page->data = mallocz(page->size);
221
-#endif
222
- }
223
-
224
- // link the free space to its page
225
- ARAL_FREE *fr = (ARAL_FREE *)page->data;
226
- fr->size = page->size;
227
- fr->page = page;
228
- fr->next = NULL;
229
- page->free_list = fr;
230
-
231
- // link the new page at the front of the list of pages
232
- DOUBLE_LINKED_LIST_PREPEND_UNSAFE(ar->internal.pages, page, prev, next);
233
-
234
- arrayalloc_free_validate_internal_check(ar, fr);
235
-}
236
-
237
-static inline void arrayalloc_lock(ARAL *ar) {
238
- if(likely(!ar->internal.lockless))
239
- netdata_spinlock_lock(&ar->internal.spinlock);
240
-}
241
-
242
-static inline void arrayalloc_unlock(ARAL *ar) {
243
- if(likely(!ar->internal.lockless))
244
- netdata_spinlock_unlock(&ar->internal.spinlock);
245
-}
246
-
247
-ARAL *arrayalloc_create(size_t element_size, size_t elements, const char *filename, char **cache_dir, bool mmap, bool lockless) {
248
- ARAL *ar = callocz(1, sizeof(ARAL));
249
- ar->requested_element_size = element_size;
250
- ar->initial_elements = elements;
251
- ar->filename = filename;
252
- ar->cache_dir = cache_dir;
253
- ar->use_mmap = mmap;
254
- ar->internal.lockless = lockless;
255
- return ar;
256
-}
257
-
258
-void arrayalloc_del_page(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
259
-
260
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(ar->internal.pages, page, prev, next);
261
-
262
- // free it
263
- if (ar->internal.mmap) {
264
- netdata_munmap(page->data, page->size);
265
-
266
- if (unlikely(unlink(page->filename) == 1))
267
- error("Cannot delete file '%s'", page->filename);
268
-
269
- freez((void *)page->filename);
270
- }
271
- else {
272
-#ifdef NETDATA_TRACE_ALLOCATIONS
273
- freez_int(page->data TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
274
-#else
275
- freez(page->data);
276
-#endif
277
- }
278
-
279
- freez(page);
280
-}
281
-
282
-void arrayalloc_destroy_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
283
- arrayalloc_lock(ar);
284
-
285
- while(ar->internal.pages)
286
- arrayalloc_del_page(ar, ar->internal.pages TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
287
-
288
- arrayalloc_unlock(ar);
289
- freez(ar);
290
-}
291
-
292
-void *arrayalloc_mallocz_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
293
-
294
- if(unlikely(!ar->internal.initialized))
295
- arrayalloc_init(ar);
296
-
297
- arrayalloc_lock(ar);
298
-
299
- if(unlikely(!ar->internal.pages || !ar->internal.pages->free_list)) {
300
-#ifdef NETDATA_ARRAYALLOC_INTERNAL_CHECKS
301
- internal_fatal(find_page_with_free_slots_internal_check(ar) != NULL,
302
- "ARRAYALLOC: first page does not have any free slots, but there is another that has!");
303
-#endif
304
- arrayalloc_add_page(ar TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
305
- }
306
-
307
- ARAL_PAGE *page = ar->internal.pages;
308
- ARAL_FREE *found_fr = page->free_list;
309
-
310
- internal_fatal(!found_fr,
311
- "ARRAYALLOC: free item to use, cannot be NULL.");
312
-
313
- internal_fatal(found_fr->size < ar->internal.element_size,
314
- "ARRAYALLOC: free item size %zu, cannot be smaller than %zu",
315
- found_fr->size, ar->internal.element_size);
316
-
317
- if(unlikely(found_fr->size - ar->internal.element_size < ar->internal.element_size)) {
318
- // we can use the entire free space entry
319
-
320
- page->free_list = found_fr->next;
321
-
322
- if(unlikely(!page->free_list)) {
323
- // we are done with this page
324
- // move the full page last
325
- // so that pages with free items remain first in the list
326
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(ar->internal.pages, page, prev, next);
327
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(ar->internal.pages, page, prev, next);
328
- }
329
- }
330
- else {
331
- // we can split the free space entry
332
-
333
- uint8_t *data = (uint8_t *)found_fr;
334
- ARAL_FREE *fr = (ARAL_FREE *)&data[ar->internal.element_size];
335
- fr->page = page;
336
- fr->size = found_fr->size - ar->internal.element_size;
337
-
338
- // link the free slot first in the page
339
- fr->next = found_fr->next;
340
- page->free_list = fr;
341
-
342
- arrayalloc_free_validate_internal_check(ar, fr);
343
- }
344
-
345
- page->used_elements++;
346
-
347
- // put the page pointer after the element
348
- uint8_t *data = (uint8_t *)found_fr;
349
- ARAL_PAGE **page_ptr = (ARAL_PAGE **)&data[ar->internal.page_ptr_offset];
350
- *page_ptr = page;
351
-
352
- arrayalloc_unlock(ar);
353
- return (void *)found_fr;
354
-}
355
-
356
-void arrayalloc_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
357
-
358
- if(unlikely(!ptr)) return;
359
- arrayalloc_lock(ar);
360
-
361
- // get the page pointer
362
- ARAL_PAGE *page;
363
- {
364
- uint8_t *data = (uint8_t *)ptr;
365
- ARAL_PAGE **page_ptr = (ARAL_PAGE **)&data[ar->internal.page_ptr_offset];
366
- page = *page_ptr;
367
-
368
-#ifdef NETDATA_INTERNAL_CHECKS
369
- // make it NULL so that we will fail on double free
370
- // do not enable this on production, because the MMAP file
371
- // will need to be saved again!
372
- *page_ptr = NULL;
373
-#endif
374
- }
375
-
376
-#ifdef NETDATA_ARRAYALLOC_INTERNAL_CHECKS
377
- {
378
- // find the page ptr belongs
379
- ARAL_PAGE *page2 = find_page_with_allocation_internal_check(ar, ptr);
380
-
381
- if(unlikely(page != page2))
382
- fatal("ARRAYALLOC: page pointers do not match!");
383
-
384
- if (unlikely(!page2))
385
- fatal("ARRAYALLOC: free of pointer %p is not in arrayalloc address space.", ptr);
386
- }
387
-#endif
388
-
389
- if(unlikely(!page))
390
- fatal("ARRAYALLOC: possible corruption or double free of pointer %p", ptr);
391
-
392
- if (unlikely(!page->used_elements))
393
- fatal("ARRAYALLOC: free of pointer %p is inside a page without any active allocations.", ptr);
394
-
395
- page->used_elements--;
396
-
397
- // make this element available
398
- ARAL_FREE *fr = (ARAL_FREE *)ptr;
399
- fr->page = page;
400
- fr->size = ar->internal.element_size;
401
- fr->next = page->free_list;
402
- page->free_list = fr;
403
-
404
- // if the page is empty, release it
405
- if(!page->used_elements)
406
- arrayalloc_del_page(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
407
-
408
- else if(page != ar->internal.pages) {
409
- // move the page with free item first
410
- // so that the next allocation will use this page
411
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(ar->internal.pages, page, prev, next);
412
- DOUBLE_LINKED_LIST_PREPEND_UNSAFE(ar->internal.pages, page, prev, next);
413
- }
414
-
415
- arrayalloc_unlock(ar);
416
-}
417
-
418
-int aral_unittest(size_t elements) {
419
- char *cache_dir = "/tmp/";
420
- ARAL *ar = arrayalloc_create(20, 10, "test-aral", &cache_dir, false, false);
421
-
422
- void *pointers[elements];
423
-
424
- for(size_t i = 0; i < elements ;i++) {
425
- pointers[i] = arrayalloc_mallocz(ar);
426
- }
427
-
428
- for(size_t div = 5; div >= 2 ;div--) {
429
- for (size_t i = 0; i < elements / div; i++) {
430
- arrayalloc_freez(ar, pointers[i]);
431
- }
432
-
433
- for (size_t i = 0; i < elements / div; i++) {
434
- pointers[i] = arrayalloc_mallocz(ar);
435
- }
436
- }
437
-
438
- for(size_t step = 50; step >= 10 ;step -= 10) {
439
- for (size_t i = 0; i < elements; i += step) {
440
- arrayalloc_freez(ar, pointers[i]);
441
- }
442
-
443
- for (size_t i = 0; i < elements; i += step) {
444
- pointers[i] = arrayalloc_mallocz(ar);
445
- }
446
- }
447
-
448
- for(size_t i = 0; i < elements ;i++) {
449
- arrayalloc_freez(ar, pointers[i]);
450
- }
451
-
452
- if(ar->internal.pages) {
453
- fprintf(stderr, "ARAL leftovers detected (1)");
454
- return 1;
455
- }
456
-
457
- size_t ops = 0; (void)ops;
458
- size_t increment = elements / 10;
459
- size_t allocated = 0;
460
- for(size_t all = increment; all <= elements ; all += increment) {
461
-
462
- for(; allocated < all ; allocated++) {
463
- pointers[allocated] = arrayalloc_mallocz(ar);
464
- ops++;
465
- }
466
-
467
- size_t to_free = now_realtime_usec() % all;
468
- size_t free_list[to_free];
469
- for(size_t i = 0; i < to_free ;i++) {
470
- size_t pos;
471
- do {
472
- pos = now_realtime_usec() % all;
473
- } while(!pointers[pos]);
474
-
475
- arrayalloc_freez(ar, pointers[pos]);
476
- pointers[pos] = NULL;
477
- free_list[i] = pos;
478
- ops++;
479
- }
480
-
481
- for(size_t i = 0; i < to_free ;i++) {
482
- size_t pos = free_list[i];
483
- pointers[pos] = arrayalloc_mallocz(ar);
484
- ops++;
485
- }
486
- }
487
-
488
- for(size_t i = 0; i < allocated - 1 ;i++) {
489
- arrayalloc_freez(ar, pointers[i]);
490
- ops++;
491
- }
492
-
493
- arrayalloc_freez(ar, pointers[allocated - 1]);
494
-
495
- if(ar->internal.pages) {
496
- fprintf(stderr, "ARAL leftovers detected (2)");
497
- return 1;
498
- }
499
-
500
- return 0;
501
-}
libnetdata/arrayalloc/arrayalloc.h
deleted
-55
@@ -1,55 +0,0 @@
1
-
2
-#ifndef ARRAYALLOC_H
3
-#define ARRAYALLOC_H 1
4
-
5
-#include "../libnetdata.h"
6
-
7
-typedef struct arrayalloc {
8
- size_t requested_element_size;
9
- size_t initial_elements;
10
- const char *filename;
11
- char **cache_dir;
12
- bool use_mmap;
13
-
14
- // private members - do not touch
15
- struct {
16
- bool mmap;
17
- bool lockless;
18
- bool initialized;
19
- size_t element_size;
20
- size_t page_ptr_offset;
21
- size_t file_number;
22
- size_t natural_page_size;
23
- size_t allocation_multiplier;
24
- size_t max_alloc_size;
25
- SPINLOCK spinlock;
26
- struct arrayalloc_page *pages;
27
- } internal;
28
-} ARAL;
29
-
30
-ARAL *arrayalloc_create(size_t element_size, size_t elements, const char *filename, char **cache_dir, bool mmap, bool lockless);
31
-int aral_unittest(size_t elements);
32
-
33
-#ifdef NETDATA_TRACE_ALLOCATIONS
34
-
35
-#define arrayalloc_mallocz(ar) arrayalloc_mallocz_internal(ar, __FILE__, __FUNCTION__, __LINE__)
36
-#define arrayalloc_freez(ar, ptr) arrayalloc_freez_internal(ar, ptr, __FILE__, __FUNCTION__, __LINE__)
37
-#define arrayalloc_destroy(ar) arrayalloc_destroy_internal(ar, __FILE__, __FUNCTION__, __LINE__)
38
-
39
-void *arrayalloc_mallocz_internal(ARAL *ar, const char *file, const char *function, size_t line);
40
-void arrayalloc_freez_internal(ARAL *ar, void *ptr, const char *file, const char *function, size_t line);
41
-void arrayalloc_destroy_internal(ARAL *ar, const char *file, const char *function, size_t line);
42
-
43
-#else // NETDATA_TRACE_ALLOCATIONS
44
-
45
-#define arrayalloc_mallocz(ar) arrayalloc_mallocz_internal(ar)
46
-#define arrayalloc_freez(ar, ptr) arrayalloc_freez_internal(ar, ptr)
47
-#define arrayalloc_destroy(ar) arrayalloc_destroy_internal(ar)
48
-
49
-void *arrayalloc_mallocz_internal(ARAL *ar);
50
-void arrayalloc_freez_internal(ARAL *ar, void *ptr);
51
-void arrayalloc_destroy_internal(ARAL *ar);
52
-
53
-#endif // NETDATA_TRACE_ALLOCATIONS
54
-
55
-#endif // ARRAYALLOC_H
libnetdata/dictionary/dictionary.c
+38
-22
@@ -789,7 +789,7 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {
789
// we didn't get a reference
790
791
if(item_is_not_referenced_and_can_be_removed(dict, item)) {
792
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next);
792
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next);
793
dict_item_free_with_hooks(dict, item);
794
deleted++;
795
@@ -1167,9 +1167,9 @@ static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item)
1167
ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
1168
1169
if(dict->options & DICT_OPTION_ADD_IN_FRONT)
1170
- DOUBLE_LINKED_LIST_PREPEND_UNSAFE(dict->items.list, item, prev, next);
1170
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(dict->items.list, item, prev, next);
1171
else
1172
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(dict->items.list, item, prev, next);
1172
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(dict->items.list, item, prev, next);
1173
1174
#ifdef NETDATA_INTERNAL_CHECKS
1175
item->ll_adder_pid = gettid();
@@ -1186,7 +1186,7 @@ static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item)
1186
static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item) {
1187
ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
1188
1189
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next);
1189
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next);
1190
1191
#ifdef NETDATA_INTERNAL_CHECKS
1192
item->ll_remover_pid = gettid();
@@ -1234,28 +1234,43 @@ static inline size_t item_get_name_len(const DICTIONARY_ITEM *item) {
1234
return strlen(item->caller_name);
1235
}
1236
1237
-static ARAL dict_items_aral = {
1238
- .filename = NULL,
1239
- .cache_dir = NULL,
1240
- .use_mmap = false,
1241
- .initial_elements = 65536 / sizeof(DICTIONARY_ITEM),
1242
- .requested_element_size = sizeof(DICTIONARY_ITEM),
1243
-};
1237
+static ARAL *dict_items_aral = NULL;
1238
+static ARAL *dict_shared_items_aral = NULL;
1239
1245
-static ARAL dict_shared_items_aral = {
1246
- .filename = NULL,
1247
- .cache_dir = NULL,
1248
- .use_mmap = false,
1249
- .initial_elements = 65536 / sizeof(DICTIONARY_ITEM_SHARED),
1250
- .requested_element_size = sizeof(DICTIONARY_ITEM_SHARED),
1251
-};
1240
+void dictionary_static_items_aral_init(void) {
1241
+ static SPINLOCK spinlock;
1242
+
1243
+ if(unlikely(!dict_items_aral || !dict_shared_items_aral)) {
1244
+ netdata_spinlock_lock(&spinlock);
1245
+
1246
+ // we have to check again
1247
+ if(!dict_items_aral)
1248
+ dict_items_aral = aral_create(
1249
+ "dict-items",
1250
+ sizeof(DICTIONARY_ITEM),
1251
+ 0,
1252
+ 4096,
1253
+ NULL, NULL, false, false);
1254
+
1255
+ // we have to check again
1256
+ if(!dict_shared_items_aral)
1257
+ dict_shared_items_aral = aral_create(
1258
+ "dict-shared-items",
1259
+ sizeof(DICTIONARY_ITEM_SHARED),
1260
+ 0,
1261
+ 4096,
1262
+ NULL, NULL, false, false);
1263
+
1264
+ netdata_spinlock_unlock(&spinlock);
1265
+ }
1266
+}
1267
1268
static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) {
1269
DICTIONARY_ITEM *item;
1270
1271
size_t size = sizeof(DICTIONARY_ITEM);
1272
// item = callocz(1, size);
1258
- item = arrayalloc_mallocz(&dict_items_aral);
1273
+ item = aral_mallocz(dict_items_aral);
1274
memset(item, 0, sizeof(DICTIONARY_ITEM));
1275
1276
#ifdef NETDATA_INTERNAL_CHECKS
@@ -1276,7 +1291,7 @@ static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t
1291
else {
1292
size = sizeof(DICTIONARY_ITEM_SHARED);
1293
// item->shared = callocz(1, size);
1279
- item->shared = arrayalloc_mallocz(&dict_shared_items_aral);
1294
+ item->shared = aral_mallocz(dict_shared_items_aral);
1295
memset(item->shared, 0, sizeof(DICTIONARY_ITEM_SHARED));
1296
1297
item->shared->links = 1;
@@ -1418,13 +1433,13 @@ static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item)
1433
value_size += item->shared->value_len;
1434
1435
// freez(item->shared);
1421
- arrayalloc_freez(&dict_shared_items_aral, item->shared);
1436
+ aral_freez(dict_shared_items_aral, item->shared);
1437
item->shared = NULL;
1438
item_size += sizeof(DICTIONARY_ITEM_SHARED);
1439
}
1440
1441
// freez(item);
1427
- arrayalloc_freez(&dict_items_aral, item);
1442
+ aral_freez(dict_items_aral, item);
1443
1444
item_size += sizeof(DICTIONARY_ITEM);
1445
@@ -1971,6 +1986,7 @@ static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dicti
1986
dict_size += reference_counter_init(dict);
1987
dict_size += hashtable_init_unsafe(dict);
1988
1989
+ dictionary_static_items_aral_init();
1990
pointer_index_init(dict);
1991
1992
DICTIONARY_STATS_PLUS_MEMORY(dict, 0, dict_size, 0);
libnetdata/july/july.c
+3
-3
@@ -64,7 +64,7 @@ void julyl_cleanup1(void) {
64
65
if(julyl_globals.protected.available_items && julyl_globals.protected.available > 10) {
66
item = julyl_globals.protected.available_items;
67
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(julyl_globals.protected.available_items, item, cache.prev, cache.next);
67
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(julyl_globals.protected.available_items, item, cache.prev, cache.next);
68
julyl_globals.protected.available--;
69
}
70
@@ -85,7 +85,7 @@ struct JulyL *julyl_get(void) {
85
86
j = julyl_globals.protected.available_items;
87
if(likely(j)) {
88
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(julyl_globals.protected.available_items, j, cache.prev, cache.next);
88
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(julyl_globals.protected.available_items, j, cache.prev, cache.next);
89
julyl_globals.protected.available--;
90
}
91
@@ -114,7 +114,7 @@ static void julyl_release(struct JulyL *j) {
114
__atomic_add_fetch(&julyl_globals.atomics.reallocs, j->reallocs, __ATOMIC_RELAXED);
115
116
netdata_spinlock_lock(&julyl_globals.protected.spinlock);
117
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(julyl_globals.protected.available_items, j, cache.prev, cache.next);
117
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(julyl_globals.protected.available_items, j, cache.prev, cache.next);
118
julyl_globals.protected.available++;
119
netdata_spinlock_unlock(&julyl_globals.protected.spinlock);
120
}
libnetdata/libnetdata.h
+70
-11
@@ -233,8 +233,9 @@ extern "C" {
233
234
// ---------------------------------------------------------------------------------------------
235
// double linked list management
236
+// inspired by https://github.com/troydhanson/uthash/blob/master/src/utlist.h
237
237
-#define DOUBLE_LINKED_LIST_PREPEND_UNSAFE(head, item, prev, next) \
238
+#define DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next) \
239
do { \
240
(item)->next = (head); \
241
\
@@ -248,7 +249,7 @@ extern "C" {
249
(head) = (item); \
250
} while (0)
251
251
-#define DOUBLE_LINKED_LIST_APPEND_UNSAFE(head, item, prev, next) \
252
+#define DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next) \
253
do { \
254
if(likely(head)) { \
255
(item)->prev = (head)->prev; \
@@ -264,39 +265,97 @@ extern "C" {
265
\
266
} while (0)
267
267
-#define DOUBLE_LINKED_LIST_REMOVE_UNSAFE(head, item, prev, next) \
268
+#define DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(head, item, prev, next) \
269
do { \
270
fatal_assert((head) != NULL); \
271
fatal_assert((item)->prev != NULL); \
272
\
272
- if((item)->prev == (item)) { \
273
+ if((item)->prev == (item)) \
274
/* it is the only item in the list */ \
275
(head) = NULL; \
275
- } \
276
+ \
277
else if((item) == (head)) { \
278
/* it is the first item */ \
279
+ fatal_assert((item)->next != NULL); \
280
(item)->next->prev = (item)->prev; \
281
(head) = (item)->next; \
282
} \
283
else { \
284
+ /* it is any other item */ \
285
(item)->prev->next = (item)->next; \
283
- if ((item)->next) { \
286
+ \
287
+ if ((item)->next) \
288
(item)->next->prev = (item)->prev; \
285
- } \
286
- else { \
289
+ else \
290
(head)->prev = (item)->prev; \
288
- } \
291
} \
292
\
293
(item)->next = NULL; \
294
(item)->prev = NULL; \
295
} while (0)
296
297
+#define DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(head, existing, item, prev, next) \
298
+ do { \
299
+ if (existing) { \
300
+ fatal_assert((head) != NULL); \
301
+ fatal_assert((item) != NULL); \
302
+ \
303
+ (item)->next = (existing); \
304
+ (item)->prev = (existing)->prev; \
305
+ (existing)->prev = (item); \
306
+ \
307
+ if ((head) == (existing)) \
308
+ (head) = (item); \
309
+ else \
310
+ (item)->prev->next = (item); \
311
+ \
312
+ } \
313
+ else \
314
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next); \
315
+ \
316
+ } while (0)
317
+
318
+#define DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(head, existing, item, prev, next) \
319
+ do { \
320
+ if (existing) { \
321
+ fatal_assert((head) != NULL); \
322
+ fatal_assert((item) != NULL); \
323
+ \
324
+ (item)->next = (existing)->next; \
325
+ (item)->prev = (existing); \
326
+ (existing)->next = (item); \
327
+ \
328
+ if ((item)->next) \
329
+ (item)->next->prev = (item); \
330
+ else \
331
+ (head)->prev = (item); \
332
+ } \
333
+ else \
334
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next); \
335
+ \
336
+ } while (0)
337
+
338
+#define DOUBLE_LINKED_LIST_APPEND_LIST_UNSAFE(head, head2, prev, next) \
339
+ do { \
340
+ if (head2) { \
341
+ if (head) { \
342
+ __typeof(head2) _head2_last_item = (head2)->prev; \
343
+ \
344
+ (head2)->prev = (head)->prev; \
345
+ (head)->prev->next = (head2); \
346
+ \
347
+ (head)->prev = _head2_last_item; \
348
+ } \
349
+ else \
350
+ (head) = (head2); \
351
+ } \
352
+ } while (0)
353
+
354
#define DOUBLE_LINKED_LIST_FOREACH_FORWARD(head, var, prev, next) \
355
for ((var) = (head); (var) ; (var) = (var)->next)
356
357
#define DOUBLE_LINKED_LIST_FOREACH_BACKWARD(head, var, prev, next) \
299
- for ((var) = (head)?(head)->prev:NULL; (var) && (var) != (head)->prev ; (var) = (var)->prev)
358
+ for ((var) = (head) ? (head)->prev : NULL ; (var) ; (var) = ((var) == (head)) ? NULL : (var)->prev)
359
360
// ---------------------------------------------------------------------------------------------
361
@@ -481,7 +540,7 @@ extern char *netdata_configured_host_prefix;
540
#include "json/json.h"
541
#include "health/health.h"
542
#include "string/utf8.h"
484
-#include "arrayalloc/arrayalloc.h"
543
+#include "libnetdata/aral/aral.h"
544
#include "onewayalloc/onewayalloc.h"
545
#include "worker_utilization/worker_utilization.h"
546
libnetdata/locks/locks.h
+1
-1
@@ -12,8 +12,8 @@ typedef pthread_mutex_t netdata_mutex_t;
12
typedef struct netdata_spinlock {
13
bool locked;
14
#ifdef NETDATA_INTERNAL_CHECKS
15
- size_t spins;
15
pid_t locker_pid;
16
+ size_t spins;
17
#endif
18
} SPINLOCK;
19
libnetdata/popen/popen.c
+3
-3
@@ -43,7 +43,7 @@ static void netdata_popen_tracking_add_pid_unsafe(pid_t pid) {
43
mp = mallocz(sizeof(struct netdata_popen));
44
mp->pid = pid;
45
46
- DOUBLE_LINKED_LIST_PREPEND_UNSAFE(netdata_popen_root, mp, prev, next);
46
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(netdata_popen_root, mp, prev, next);
47
}
48
49
// myp_del deletes pid if we're tracking.
@@ -61,7 +61,7 @@ static void netdata_popen_tracking_del_pid(pid_t pid) {
61
}
62
63
if(mp) {
64
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(netdata_popen_root, mp, prev, next);
64
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(netdata_popen_root, mp, prev, next);
65
freez(mp);
66
}
67
else
@@ -96,7 +96,7 @@ void netdata_popen_tracking_cleanup(void) {
96
97
while(netdata_popen_root) {
98
struct netdata_popen *mp = netdata_popen_root;
99
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(netdata_popen_root, mp, prev, next);
99
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(netdata_popen_root, mp, prev, next);
100
freez(mp);
101
}
102
libnetdata/worker_utilization/worker_utilization.c
+2
-2
@@ -100,7 +100,7 @@ void worker_register(const char *name) {
100
}
101
102
netdata_spinlock_lock(&workname->spinlock);
103
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(workname->base, worker, prev, next);
103
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(workname->base, worker, prev, next);
104
netdata_spinlock_unlock(&workname->spinlock);
105
106
netdata_spinlock_unlock(&workers_globals.spinlock);
@@ -141,7 +141,7 @@ void worker_unregister(void) {
141
if(PValue) {
142
struct workers_workname *workname = *PValue;
143
netdata_spinlock_lock(&workname->spinlock);
144
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(workname->base, worker, prev, next);
144
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(workname->base, worker, prev, next);
145
netdata_spinlock_unlock(&workname->spinlock);
146
147
if(!workname->base) {
streaming/replication.c
+1
-1
@@ -392,7 +392,7 @@ static void replication_query_execute(BUFFER *wb, struct replication_query *q, s
392
if (likely( d->sp.start_time_s <= min_end_time &&
393
d->sp.end_time_s >= min_end_time &&
394
!storage_point_is_unset(d->sp) &&
395
- !storage_point_is_empty(d->sp))) {
395
+ !storage_point_is_gap(d->sp))) {
396
397
buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_SET " \"%s\" " NETDATA_DOUBLE_FORMAT " \"%s\"\n",
398
rrddim_id(d->rd), d->sp.sum, d->sp.flags & SN_FLAG_RESET ? "R" : "");
streaming/rrdpush.c
+4
-4
@@ -498,8 +498,8 @@ int connect_to_one_of_destinations(
498
// move the current item to the end of the list
499
// without this, this destination will break the loop again and again
500
// not advancing the destinations to find one that may work
501
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->destinations, d, prev, next);
502
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->destinations, d, prev, next);
501
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->destinations, d, prev, next);
502
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(host->destinations, d, prev, next);
503
504
break;
505
}
@@ -522,7 +522,7 @@ bool destinations_init_add_one(char *entry, void *data) {
522
523
__atomic_add_fetch(&netdata_buffers_statistics.rrdhost_senders, sizeof(struct rrdpush_destinations), __ATOMIC_RELAXED);
524
525
- DOUBLE_LINKED_LIST_APPEND_UNSAFE(t->list, d, prev, next);
525
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(t->list, d, prev, next);
526
527
t->count++;
528
info("STREAM: added streaming destination No %d: '%s' to host '%s'", t->count, string2str(d->destination), rrdhost_hostname(t->host));
@@ -549,7 +549,7 @@ void rrdpush_destinations_init(RRDHOST *host) {
549
void rrdpush_destinations_free(RRDHOST *host) {
550
while (host->destinations) {
551
struct rrdpush_destinations *tmp = host->destinations;
552
- DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->destinations, tmp, prev, next);
552
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->destinations, tmp, prev, next);
553
string_freez(tmp->destination);
554
freez(tmp);
555
__atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_senders, sizeof(struct rrdpush_destinations), __ATOMIC_RELAXED);
web/api/queries/query.c
+113
-17
@@ -17,7 +17,7 @@
17
#include "percentile/percentile.h"
18
#include "trimmed_mean/trimmed_mean.h"
19
20
-#define POINTS_TO_EXPAND_QUERY 0
20
+#define POINTS_TO_EXPAND_QUERY 5
21
22
// ----------------------------------------------------------------------------
23
@@ -927,6 +927,8 @@ typedef struct query_engine_ops {
927
// query planer
928
size_t current_plan;
929
time_t current_plan_expire_time;
930
+ time_t plan_expanded_after;
931
+ time_t plan_expanded_before;
932
933
// storage queries
934
size_t tier;
@@ -955,6 +957,20 @@ typedef struct query_engine_ops {
957
958
#define query_plan_should_switch_plan(ops, now) ((now) >= (ops)->current_plan_expire_time)
959
960
+static size_t query_planer_expand_duration_in_points(time_t this_update_every, time_t next_update_every) {
961
+
962
+ time_t delta = this_update_every - next_update_every;
963
+ if(delta < 0) delta = -delta;
964
+
965
+ size_t points;
966
+ if(delta < this_update_every * POINTS_TO_EXPAND_QUERY)
967
+ points = POINTS_TO_EXPAND_QUERY;
968
+ else
969
+ points = (delta + this_update_every - 1) / this_update_every;
970
+
971
+ return points;
972
+}
973
+
974
static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) {
975
QUERY_METRIC *qm = ops->qm;
976
@@ -962,8 +978,35 @@ static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) {
978
size_t tier = qm->plan.array[p].tier;
979
time_t update_every = qm->tiers[tier].db_update_every_s;
980
965
- time_t after = qm->plan.array[p].after - (update_every * POINTS_TO_EXPAND_QUERY);
966
- time_t before = qm->plan.array[p].before + (update_every * POINTS_TO_EXPAND_QUERY);
981
+ size_t points_to_add_to_after;
982
+ if(p > 0) {
983
+ // there is another plan before to this
984
+
985
+ size_t tier0 = qm->plan.array[p - 1].tier;
986
+ time_t update_every0 = qm->tiers[tier0].db_update_every_s;
987
+
988
+ points_to_add_to_after = query_planer_expand_duration_in_points(update_every, update_every0);
989
+ }
990
+ else
991
+ points_to_add_to_after = (tier == 0) ? 0 : POINTS_TO_EXPAND_QUERY;
992
+
993
+ size_t points_to_add_to_before;
994
+ if(p + 1 < qm->plan.used) {
995
+ // there is another plan after to this
996
+
997
+ size_t tier1 = qm->plan.array[p+1].tier;
998
+ time_t update_every1 = qm->tiers[tier1].db_update_every_s;
999
+
1000
+ points_to_add_to_before = query_planer_expand_duration_in_points(update_every, update_every1);
1001
+ }
1002
+ else
1003
+ points_to_add_to_before = POINTS_TO_EXPAND_QUERY;
1004
+
1005
+ time_t after = qm->plan.array[p].after - (time_t)(update_every * points_to_add_to_after);
1006
+ time_t before = qm->plan.array[p].before + (time_t)(update_every * points_to_add_to_before);
1007
+
1008
+ qm->plan.array[p].expanded_after = after;
1009
+ qm->plan.array[p].expanded_before = before;
1010
1011
struct query_metric_tier *tier_ptr = &qm->tiers[tier];
1012
tier_ptr->eng->api.query_ops.init(
@@ -1027,9 +1070,12 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
1070
ops->current_plan_expire_time = qm->plan.array[plan_id + 1].after;
1071
else
1072
ops->current_plan_expire_time = qm->plan.array[plan_id].before;
1073
+
1074
+ ops->plan_expanded_after = qm->plan.array[plan_id].expanded_after;
1075
+ ops->plan_expanded_before = qm->plan.array[plan_id].expanded_before;
1076
}
1077
1032
-static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) {
1078
+static bool query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) {
1079
QUERY_METRIC *qm = ops->qm;
1080
1081
size_t old_plan = ops->current_plan;
@@ -1043,7 +1089,7 @@ static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t las
1089
ops->current_plan_expire_time = ops->r->internal.qt->window.before;
1090
// let the query run with current plan
1091
// we will not switch it
1046
- return;
1092
+ return false;
1093
}
1094
1095
next_plan_before_time = qm->plan.array[ops->current_plan].before;
@@ -1052,11 +1098,12 @@ static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t las
1098
if(!query_metric_is_valid_tier(qm, qm->plan.array[ops->current_plan].tier)) {
1099
ops->current_plan = old_plan;
1100
ops->current_plan_expire_time = ops->r->internal.qt->window.before;
1055
- return;
1101
+ return false;
1102
}
1103
1104
query_planer_finalize_plan(ops, old_plan);
1105
query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
1106
+ return true;
1107
}
1108
1109
static int compare_query_plan_entries_on_start_time(const void *a, const void *b) {
@@ -1273,6 +1320,11 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1320
QUERY_POINT last1_point = QUERY_POINT_EMPTY;
1321
QUERY_POINT new_point = QUERY_POINT_EMPTY;
1322
1323
+ // ONE POINT READ-AHEAD
1324
+ // when we switch plans, we read-ahead a point from the next plan
1325
+ // to join them smoothly at the exact time the next plan begins
1326
+ STORAGE_POINT next1_point = STORAGE_POINT_UNSET;
1327
+
1328
time_t now_start_time = after_wanted - ops->query_granularity;
1329
time_t now_end_time = after_wanted + ops->view_update_every - ops->query_granularity;
1330
@@ -1311,8 +1363,41 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1363
1364
// fetch the new point
1365
{
1314
- db_points_read_since_plan_switch++;
1315
- STORAGE_POINT sp = ops->next_metric(ops->handle);
1366
+ STORAGE_POINT sp;
1367
+ if(likely(storage_point_is_unset(next1_point))) {
1368
+ db_points_read_since_plan_switch++;
1369
+ sp = ops->next_metric(ops->handle);
1370
+ }
1371
+ else {
1372
+ // ONE POINT READ-AHEAD
1373
+ sp = next1_point;
1374
+ storage_point_unset(next1_point);
1375
+ db_points_read_since_plan_switch = 1;
1376
+ }
1377
+
1378
+ // ONE POINT READ-AHEAD
1379
+ if(unlikely(query_plan_should_switch_plan(ops, sp.end_time_s) &&
1380
+ query_planer_next_plan(ops, now_end_time, new_point.end_time))) {
1381
+
1382
+ // The end time of the current point, crosses our plans (tiers)
1383
+ // so, we switched plan (tier)
1384
+ //
1385
+ // There are 2 cases now:
1386
+ //
1387
+ // A. the entire point of the previous plan is to the future of point from the next plan
1388
+ // B. part of the point of the previous plan overlaps with the point from the next plan
1389
+
1390
+ STORAGE_POINT sp2 = ops->next_metric(ops->handle);
1391
+
1392
+ if(sp.start_time_s > sp2.start_time_s)
1393
+ // the point from the previous plan is useless
1394
+ sp = sp2;
1395
+ else
1396
+ // let the query run from the previous plan
1397
+ // but setting this will also cut off the interpolation
1398
+ // of the point from the previous plan
1399
+ next1_point = sp2;
1400
+ }
1401
1402
ops->db_points_read_per_tier[ops->tier]++;
1403
ops->db_total_points_read++;
@@ -1326,8 +1411,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1411
// info("QUERY: got point %zu, from time %ld to %ld // now from %ld to %ld // query from %ld to %ld",
1412
// new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
1413
//
1329
- // set the right value to the point we got
1330
- if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
1414
+ // get the right value from the point we got
1415
+ if(likely(!storage_point_is_unset(sp) && !storage_point_is_gap(sp))) {
1416
1417
if(unlikely(use_anomaly_bit_as_value))
1418
new_point.value = new_point.anomaly;
@@ -1391,7 +1476,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1476
count_same_end_time = 0;
1477
1478
// decide how to use this point
1394
- if(likely(new_point.end_time < now_end_time)) { // likely to favor tier0
1479
+ if(likely(new_point.end_time <= now_end_time)) { // likely to favor tier0
1480
// this db point ends before our now_end_time
1481
1482
if(likely(new_point.end_time >= now_start_time)) { // likely to favor tier0
@@ -1408,8 +1493,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1493
// at exactly the time we will want
1494
1495
// we only log if this is not point 1
1411
- internal_error(new_point.end_time < after_wanted &&
1412
- new_point.id > POINTS_TO_EXPAND_QUERY + 1,
1496
+ internal_error(new_point.end_time < ops->plan_expanded_after &&
1497
+ db_points_read_since_plan_switch > 1,
1498
"QUERY: '%s', dimension '%s' next_metric() "
1499
"returned point %zu from %ld time %ld, "
1500
"which is entirely before our current timeframe %ld to %ld "
@@ -1417,7 +1502,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1502
qt->id, string2str(qm->dimension.id),
1503
new_point.id, new_point.start_time, new_point.end_time,
1504
now_start_time, now_end_time,
1420
- after_wanted, before_wanted);
1505
+ ops->plan_expanded_after, ops->plan_expanded_before);
1506
}
1507
1508
}
@@ -1430,19 +1515,30 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1515
1516
if(unlikely(count_same_end_time)) {
1517
internal_error(true,
1433
- "QUERY: '%s', dimension '%s', the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",
1434
- qt->id, string2str(qm->dimension.id), last1_point.end_time, count_same_end_time);
1518
+ "QUERY: '%s', dimension '%s', the database does not advance the query,"
1519
+ " it returned an end time less or equal to the end time of the last "
1520
+ "point we got %ld, %zu times",
1521
+ qt->id, string2str(qm->dimension.id),
1522
+ last1_point.end_time, count_same_end_time);
1523
1524
if(unlikely(new_point.end_time <= last1_point.end_time))
1525
new_point.end_time = now_end_time;
1526
}
1527
1528
+ time_t stop_time = new_point.end_time;
1529
+ if(unlikely(!storage_point_is_unset(next1_point))) {
1530
+ // ONE POINT READ-AHEAD
1531
+ // the point crosses the start time of the
1532
+ // read ahead storage point we have read
1533
+ stop_time = next1_point.start_time_s;
1534
+ }
1535
+
1536
// the inner loop
1537
// we have 3 points in memory: last2, last1, new
1538
// we select the one to use based on their timestamps
1539
1540
size_t iterations = 0;
1445
- for ( ; now_end_time <= new_point.end_time && points_added < points_wanted ;
1541
+ for ( ; now_end_time <= stop_time && points_added < points_wanted ;
1542
now_end_time += ops->view_update_every, iterations++) {
1543
1544
// now_start_time is wrong in this loop
web/server/web_client.c
+10
-11
@@ -129,10 +129,10 @@ void web_client_request_done(struct web_client *w) {
129
, mode
130
, sent
131
, size
132
- , -((size > 0) ? ((size - sent) / (double) size * 100.0) : 0.0)
133
- , dt_usec(&w->tv_ready, &w->tv_in) / 1000.0
134
- , dt_usec(&tv, &w->tv_ready) / 1000.0
135
- , dt_usec(&tv, &w->tv_in) / 1000.0
132
+ , -((size > 0) ? ((double)(size - sent) / (double) size * 100.0) : 0.0)
133
+ , (double)dt_usec(&w->tv_ready, &w->tv_in) / 1000.0
134
+ , (double)dt_usec(&tv, &w->tv_ready) / 1000.0
135
+ , (double)dt_usec(&tv, &w->tv_in) / 1000.0
136
, w->response.code
137
, strip_control_characters(w->last_url)
138
);
@@ -302,7 +302,7 @@ int mysendfile(struct web_client *w, char *filename) {
302
}
303
}
304
305
- // if the filename contains a .. refuse to serve it
305
+ // if the filename contains a double dot refuse to serve it
306
if(strstr(filename, "..") != 0) {
307
debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
308
w->response.data->contenttype = CT_TEXT_HTML;
@@ -831,9 +831,8 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
831
* @param s is the first address of the string.
832
* @param ptr is the address of the separator.
833
*/
834
-static void web_client_set_path_query(struct web_client *w, char *s, char *ptr) {
834
+static void web_client_set_path_query(struct web_client *w, const char *s, char *ptr) {
835
w->url_path_length = (size_t)(ptr -s);
836
-
836
w->url_search_path = ptr;
837
}
838
@@ -1429,7 +1428,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1428
// replace the zero bytes with spaces
1429
buffer_char_replace(w->response.data, '\0', ' ');
1430
1432
- // just leave the buffer as is
1431
+ // just leave the buffer as-is
1432
// it will be copied back to the client
1433
1434
return HTTP_RESP_OK;
@@ -1546,7 +1545,7 @@ void web_client_process_request(struct web_client *w) {
1545
break;
1546
}
1547
1549
- // keep track of the time we done processing
1548
+ // keep track of the processing time
1549
now_realtime_timeval(&w->tv_ready);
1550
1551
w->response.sent = 0;
@@ -1847,7 +1846,7 @@ ssize_t web_client_read_file(struct web_client *w)
1846
if(unlikely(w->response.rlen <= w->response.data->len))
1847
return 0;
1848
1850
- ssize_t left = w->response.rlen - w->response.data->len;
1849
+ ssize_t left = (ssize_t)(w->response.rlen - w->response.data->len);
1850
ssize_t bytes = read(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t)left);
1851
if(likely(bytes > 0)) {
1852
size_t old = w->response.data->len;
@@ -1897,7 +1896,7 @@ ssize_t web_client_receive(struct web_client *w)
1896
return web_client_read_file(w);
1897
1898
ssize_t bytes;
1900
- ssize_t left = w->response.data->size - w->response.data->len;
1899
+ ssize_t left = (ssize_t)(w->response.data->size - w->response.data->len);
1900
1901
// do we have any space for more data?
1902
buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_RECEIVE_SIZE);