@cryptotaxi247 / netdata-1 / commits / 091540d59

feat(dbengine): make dbengine page cache undumpable and dedupuble (#12765)

* make netdata more awesome * reworked on-madvise and mmap to provide clarity

Ilya Mashchenko committed Apr 28, 2022 at 11:19 UTC 091540d59aaf80c8198c5a7fbd5440353d82d7ff
8 files changed +117 -97
database/engine/pagecache.c
+2 -2
@@ -356,7 +356,7 @@ static void pg_cache_evict_unsafe(struct rrdengine_instance *ctx, struct rrdeng_
356 {
357 struct page_cache_descr *pg_cache_descr = descr->pg_cache_descr;
358
359 - freez(pg_cache_descr->page);
359 + dbengine_page_free(pg_cache_descr->page);
360 pg_cache_descr->page = NULL;
361 pg_cache_descr->flags &= ~RRD_PAGE_POPULATED;
362 pg_cache_release_pages_unsafe(ctx, 1);
@@ -1222,7 +1222,7 @@ void free_page_cache(struct rrdengine_instance *ctx)
1222 /* Check rrdenglocking.c */
1223 pg_cache_descr = descr->pg_cache_descr;
1224 if (pg_cache_descr->flags & RRD_PAGE_POPULATED) {
1225 - freez(pg_cache_descr->page);
1225 + dbengine_page_free(pg_cache_descr->page);
1226 bytes_freed += RRDENG_BLOCK_SIZE;
1227 }
1228 rrdeng_destroy_pg_cache_descr(ctx, pg_cache_descr);
database/engine/rrdengine.c
+12 -2
@@ -11,6 +11,16 @@ rrdeng_stats_t global_flushing_pressure_page_deletions = 0;
11
12 static unsigned pages_per_extent = MAX_PAGES_PER_EXTENT;
13
14 +void *dbengine_page_alloc() {
15 + void *page = netdata_mmap(NULL, RRDENG_BLOCK_SIZE, MAP_PRIVATE, enable_ksm);
16 + if(!page) fatal("Cannot allocate dbengine page cache page, with mmap()");
17 + return page;
18 +}
19 +
20 +void dbengine_page_free(void *page) {
21 + munmap(page, RRDENG_BLOCK_SIZE);
22 +}
23 +
24 static void sanity_check(void)
25 {
26 /* Magic numbers must fit in the super-blocks */
@@ -176,7 +186,7 @@ void read_cached_extent_cb(struct rrdengine_worker_config* wc, unsigned idx, str
186 struct extent_info *extent = xt_io_descr->descr_array[0]->extent;
187
188 for (i = 0 ; i < xt_io_descr->descr_count; ++i) {
179 - page = mallocz(RRDENG_BLOCK_SIZE);
189 + page = dbengine_page_alloc();
190 descr = xt_io_descr->descr_array[i];
191 for (j = 0, page_offset = 0 ; j < extent->number_of_pages ; ++j) {
192 /* care, we don't hold the descriptor mutex */
@@ -331,7 +341,7 @@ after_crc_check:
341 continue; /* Failed to reserve a suitable page */
342 is_prefetched_page = 1;
343 }
334 - page = mallocz(RRDENG_BLOCK_SIZE);
344 + page = dbengine_page_alloc();
345
346 /* care, we don't hold the descriptor mutex */
347 if (have_read_error) {
database/engine/rrdengine.h
+3
@@ -227,6 +227,9 @@ struct rrdengine_instance {
227 struct rrdengine_statistics stats;
228 };
229
230 +extern void *dbengine_page_alloc(void);
231 +extern void dbengine_page_free(void *page);
232 +
233 extern int init_rrd_files(struct rrdengine_instance *ctx);
234 extern void finalize_rrd_files(struct rrdengine_instance *ctx);
235 extern void rrdeng_test_quota(struct rrdengine_worker_config* wc);
database/engine/rrdengineapi.c
+2 -2
@@ -202,7 +202,7 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
202 /* handle->prev_descr = descr;*/
203 }
204 } else {
205 - freez(descr->pg_cache_descr->page);
205 + dbengine_page_free(descr->pg_cache_descr->page);
206 rrdeng_destroy_pg_cache_descr(ctx, descr->pg_cache_descr);
207 freez(descr);
208 }
@@ -724,7 +724,7 @@ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrde
724
725 descr = pg_cache_create_descr();
726 descr->id = id; /* TODO: add page type: metric, log, something? */
727 - page = mallocz(RRDENG_BLOCK_SIZE); /*TODO: add page size */
727 + page = dbengine_page_alloc(); /*TODO: add page size */
728 rrdeng_page_descr_mutex_lock(ctx, descr);
729 pg_cache_descr = descr->pg_cache_descr;
730 pg_cache_descr->page = page;
database/rrddim.c
+5 -6
@@ -244,12 +244,11 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
244
245 if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP ||
246 memory_mode == RRD_MEMORY_MODE_RAM) {
247 - rd = (RRDDIM *)mymmap(
248 - (memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename
249 - , size
250 - , ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE)
251 - , 1
252 - );
247 + rd = (RRDDIM *)netdata_mmap(
248 + (memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename,
249 + size,
250 + ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
251 + 1);
252
253 if(likely(rd)) {
254 // we have a file mapped for rd
database/rrdset.c
+5 -6
@@ -744,12 +744,11 @@ RRDSET *rrdset_create_custom(
744 snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
745 if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP ||
746 memory_mode == RRD_MEMORY_MODE_RAM) {
747 - st = (RRDSET *) mymmap(
748 - (memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename
749 - , size
750 - , ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE)
751 - , 0
752 - );
747 + st = (RRDSET *)netdata_mmap(
748 + (memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename,
749 + size,
750 + ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
751 + 0);
752
753 if(st) {
754 memset(&st->avl, 0, sizeof(avl_t));
libnetdata/libnetdata.c
+87 -78
@@ -11,7 +11,12 @@
11 #endif /* __FreeBSD__ || __APPLE__*/
12
13 struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };
14 +
15 +#ifdef MADV_MERGEABLE
16 int enable_ksm = 1;
17 +#else
18 +int enable_ksm = 0;
19 +#endif
20
21 volatile sig_atomic_t netdata_exit = 0;
22 const char *program_version = VERSION;
@@ -939,108 +944,112 @@ static int memory_file_open(const char *filename, size_t size) {
944 return fd;
945 }
946
942 -// mmap_shared is used for memory mode = map
943 -static void *memory_file_mmap(const char *filename, size_t size, int flags) {
944 - // info("memory_file_mmap('%s', %zu", filename, size);
945 - static int log_madvise = 1;
947 +static inline int madvise_sequential(void *mem, size_t len) {
948 + static int logger = 1;
949 + int ret = madvise(mem, len, MADV_SEQUENTIAL);
950
947 - int fd = -1;
948 - if(filename) {
949 - fd = memory_file_open(filename, size);
950 - if(fd == -1) return MAP_FAILED;
951 - }
951 + if (ret != 0 && logger-- > 0) error("madvise(MADV_SEQUENTIAL) failed.");
952 + return ret;
953 +}
954 +static inline int madvise_dontfork(void *mem, size_t len) {
955 + static int logger = 1;
956 + int ret = madvise(mem, len, MADV_DONTFORK);
957
953 - void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0);
954 - if (mem != MAP_FAILED) {
955 -#ifdef NETDATA_LOG_ALLOCATIONS
956 - mmap_accounting(size);
957 -#endif
958 - int advise = MADV_SEQUENTIAL | MADV_DONTFORK;
959 - if (flags & MAP_SHARED) advise |= MADV_WILLNEED;
958 + if (ret != 0 && logger-- > 0) error("madvise(MADV_DONTFORK) failed.");
959 + return ret;
960 +}
961 +static inline int madvise_willneed(void *mem, size_t len) {
962 + static int logger = 1;
963 + int ret = madvise(mem, len, MADV_WILLNEED);
964
961 - if (madvise(mem, size, advise) != 0 && log_madvise) {
962 - error("Cannot advise the kernel about shared memory usage.");
963 - log_madvise--;
964 - }
965 - }
965 + if (ret != 0 && logger-- > 0) error("madvise(MADV_WILLNEED) failed.");
966 + return ret;
967 +}
968 +static inline int madvise_dontdump(void *mem, size_t len) {
969 + static int logger = 1;
970 + int ret = madvise(mem, len, MADV_DONTDUMP);
971
967 - if(fd != -1)
968 - close(fd);
972 + if (ret != 0 && logger-- > 0) error("madvise(MADV_DONTDUMP) failed.");
973 + return ret;
974 +}
975 +static inline int madvise_mergeable(void *mem, size_t len) {
976 +#ifdef MADV_MERGEABLE
977 + static int logger = 1;
978 + int ret = madvise(mem, len, MADV_MERGEABLE);
979
970 - return mem;
980 + if (ret != 0 && logger-- > 0) error("madvise(MADV_MERGEABLE) failed.");
981 + return ret;
982 +#else
983 + return 0;
984 +#endif
985 }
986
973 -#ifdef MADV_MERGEABLE
974 -static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) {
975 - // info("memory_file_mmap_ksm('%s', %zu", filename, size);
976 - static int log_madvise_2 = 1, log_madvise_3 = 1;
987 +void *netdata_mmap(const char *filename, size_t size, int flags, int ksm) {
988 + // info("netdata_mmap('%s', %zu", filename, size);
989 +
990 + // MAP_SHARED is used in memory mode map
991 + // MAP_PRIVATE is used in memory mode ram and save
992 +
993 + if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE)))
994 + fatal("Neither MAP_SHARED or MAP_PRIVATE were given to netdata_mmap()");
995 +
996 + if(unlikely((flags & MAP_SHARED) && (flags & MAP_PRIVATE)))
997 + fatal("Both MAP_SHARED and MAP_PRIVATE were given to netdata_mmap()");
998 +
999 + if(unlikely((flags & MAP_SHARED) && (!filename || !*filename)))
1000 + fatal("MAP_SHARED requested, without a filename to netdata_mmap()");
1001 +
1002 + // don't enable ksm is the global setting is disabled
1003 + if(unlikely(!enable_ksm)) ksm = 0;
1004 +
1005 + // KSM only merges anonymous (private) pages, never pagecache (file) pages
1006 + // but MAP_PRIVATE without MAP_ANONYMOUS it fails too, so we need it always
1007 + if((flags & MAP_PRIVATE)) flags |= MAP_ANONYMOUS;
1008
1009 int fd = -1;
979 - if(filename) {
1010 + void *mem = MAP_FAILED;
1011 +
1012 + if(filename && *filename) {
1013 + // open/create the file to be used
1014 fd = memory_file_open(filename, size);
981 - if(fd == -1) return MAP_FAILED;
1015 + if(fd == -1) goto cleanup;
1016 }
1017
984 - void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0);
1018 + int fd_for_mmap = fd;
1019 + if(fd != -1 && (flags & MAP_PRIVATE)) {
1020 + // this is MAP_PRIVATE allocation
1021 + // no need for mmap() to use our fd
1022 + // we will copy the file into the memory allocated
1023 + fd_for_mmap = -1;
1024 + }
1025 +
1026 + mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd_for_mmap, 0);
1027 if (mem != MAP_FAILED) {
1028 +
1029 #ifdef NETDATA_LOG_ALLOCATIONS
1030 mmap_accounting(size);
1031 #endif
989 - if(fd != -1) {
1032 +
1033 + // if we have a file open, but we didn't give it to mmap(),
1034 + // we have to read the file into the memory block we allocated
1035 + if(fd != -1 && fd_for_mmap == -1) {
1036 if (lseek(fd, 0, SEEK_SET) == 0) {
1037 if (read(fd, mem, size) != (ssize_t) size)
992 - error("Cannot read from file '%s'", filename);
1038 + info("Cannot read from file '%s'", filename);
1039 }
994 - else error("Cannot seek to beginning of file '%s'.", filename);
1040 + else info("Cannot seek to beginning of file '%s'.", filename);
1041 }
1042
997 - // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE
998 - if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) {
999 - error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.", filename);
1000 - log_madvise_2--;
1001 - }
1002 -
1003 - if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) {
1004 - error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.", filename);
1005 - log_madvise_3--;
1006 - }
1043 + madvise_sequential(mem, size);
1044 + madvise_dontfork(mem, size);
1045 + madvise_dontdump(mem, size);
1046 + if(flags & MAP_SHARED) madvise_willneed(mem, size);
1047 + if(ksm) madvise_mergeable(mem, size);
1048 }
1049
1009 - if(fd != -1)
1010 - close(fd);
1011 -
1012 - return mem;
1013 -}
1014 -#else
1015 -static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) {
1016 - // info("memory_file_mmap_ksm FALLBACK ('%s', %zu", filename, size);
1017 -
1018 - if(filename)
1019 - return memory_file_mmap(filename, size, flags);
1020 -
1021 - // when KSM is not available and no filename is given (memory mode = ram),
1022 - // we just report failure
1023 - return MAP_FAILED;
1024 -}
1025 -#endif
1026 -
1027 -void *mymmap(const char *filename, size_t size, int flags, int ksm) {
1028 - void *mem = NULL;
1029 -
1030 - if (filename && (flags & MAP_SHARED || !enable_ksm || !ksm))
1031 - // memory mode = map | save
1032 - // when KSM is not enabled
1033 - // MAP_SHARED is used for memory mode = map (no KSM possible)
1034 - mem = memory_file_mmap(filename, size, flags);
1035 -
1036 - else
1037 - // memory mode = save | ram
1038 - // when KSM is enabled
1039 - // for memory mode = ram, the filename is NULL
1040 - mem = memory_file_mmap_ksm(filename, size, flags);
1041 -
1050 +cleanup:
1051 + if(fd != -1) close(fd);
1052 if(mem == MAP_FAILED) return NULL;
1043 -
1053 errno = 0;
1054 return mem;
1055 }
libnetdata/libnetdata.h
+1 -1
@@ -250,7 +250,7 @@ extern void freez(void *ptr);
250 extern void json_escape_string(char *dst, const char *src, size_t size);
251 extern void json_fix_string(char *s);
252
253 -extern void *mymmap(const char *filename, size_t size, int flags, int ksm);
253 +extern void *netdata_mmap(const char *filename, size_t size, int flags, int ksm);
254 extern int memory_file_save(const char *filename, void *mem, size_t size);
255
256 extern int fd_is_valid(int fd);