@cryptotaxi247 / netdata-1 / commits / 4f072e604

Unified memory API (#19396)

* split memory management into separate files - no functional changes * always use nd_mmap() and unified memory alignment logic * off_t instead of __off_t * added windows version of get_system_pagesize()

Costa Tsaousis committed Jan 14, 2025 at 14:15 UTC 4f072e6040a196519dcac5cda8449a4abc2efee2

19 files changed +870 -795

CMakeLists.txt

@@ -980,6 +980,13 @@ set(LIBNETDATA_FILES

980 src/libnetdata/locks/benchmark-rw.c

981 src/libnetdata/locks/benchmark-rw.h

982 src/libnetdata/log/nd_log-libunwind.c

983 + src/libnetdata/memory/nd-mallocz.c

984 + src/libnetdata/memory/nd-mallocz.h

985 + src/libnetdata/memory/nd-mmap.c

986 + src/libnetdata/memory/nd-mmap.h

987 + src/libnetdata/memory/alignment.h

988 + src/libnetdata/os/get_system_pagesize.c

989 + src/libnetdata/os/get_system_pagesize.h

990 )

991

992 set(LIBH2O_FILES

src/collectors/cgroups.plugin/cgroup-discovery.c

+4 -5

@@ -1070,9 +1070,8 @@ static void netdata_cgroup_ebpf_initialize_shm()

1070 goto end_init_shm;

1071 }

1072

1073 - shm_cgroup_ebpf.header = (netdata_ebpf_cgroup_shm_header_t *) mmap(NULL, length,

1074 - PROT_READ | PROT_WRITE, MAP_SHARED,

1075 - shm_fd_cgroup_ebpf, 0);

1073 + shm_cgroup_ebpf.header = (netdata_ebpf_cgroup_shm_header_t *)

1074 + nd_mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd_cgroup_ebpf, 0);

1075

1076 if (unlikely(MAP_FAILED == shm_cgroup_ebpf.header)) {

1077 shm_cgroup_ebpf.header = NULL;

@@ -1091,7 +1090,7 @@ static void netdata_cgroup_ebpf_initialize_shm()

1090 }

1091

1092 collector_error("Cannot create semaphore, integration between eBPF and cgroup won't happen");

1094 - munmap(shm_cgroup_ebpf.header, length);

1093 + nd_munmap(shm_cgroup_ebpf.header, length);

1094 shm_cgroup_ebpf.header = NULL;

1095

1096 end_init_shm:

@@ -1108,7 +1107,7 @@ static void cgroup_cleanup_ebpf_integration()

1107

1108 if (shm_cgroup_ebpf.header) {

1109 shm_cgroup_ebpf.header->cgroup_root_count = 0;

1111 - munmap(shm_cgroup_ebpf.header, shm_cgroup_ebpf.header->body_length);

1110 + nd_munmap(shm_cgroup_ebpf.header, shm_cgroup_ebpf.header->body_length);

1111 }

1112

1113 if (shm_fd_cgroup_ebpf > 0) {

src/collectors/ebpf.plugin/ebpf_cgroup.c

+4 -4

@@ -26,7 +26,7 @@ static inline void *ebpf_cgroup_map_shm_locally(int fd, size_t length)

26 {

27 void *value;

29 - value = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

29 + value = nd_mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

30 if (!value) {

31 netdata_log_error("Cannot map shared memory used between eBPF and cgroup, integration between processes won't happen");

32 close(shm_fd_ebpf_cgroup);

@@ -44,7 +44,7 @@ static inline void *ebpf_cgroup_map_shm_locally(int fd, size_t length)

44 */

45 void ebpf_unmap_cgroup_shared_memory()

46 {

47 - munmap(ebpf_mapped_memory, shm_ebpf_cgroup.header->body_length);

47 + nd_munmap(ebpf_mapped_memory, shm_ebpf_cgroup.header->body_length);

48 }

50 /**

@@ -87,7 +87,7 @@ void ebpf_map_cgroup_shared_memory()

88 size_t length = header->body_length;

90 - munmap(header, sizeof(netdata_ebpf_cgroup_shm_header_t));

90 + nd_munmap(header, sizeof(netdata_ebpf_cgroup_shm_header_t));

92 if (length <= ((sizeof(netdata_ebpf_cgroup_shm_header_t) + sizeof(netdata_ebpf_cgroup_shm_body_t)))) {

93 return;

@@ -105,7 +105,7 @@ void ebpf_map_cgroup_shared_memory()

105 if (shm_sem_ebpf_cgroup == SEM_FAILED) {

106 netdata_log_error("Cannot create semaphore, integration between eBPF and cgroup won't happen");

107 limit_try = NETDATA_EBPF_CGROUP_MAX_TRIES + 1;

108 - munmap(ebpf_mapped_memory, length);

108 + nd_munmap(ebpf_mapped_memory, length);

109 shm_ebpf_cgroup.header = NULL;

110 shm_ebpf_cgroup.body = NULL;

111 close(shm_fd_ebpf_cgroup);

src/daemon/pulse/pulse-daemon-memory-system.c

+1 -1

@@ -229,7 +229,7 @@ void pulse_daemon_memory_system_do(bool extended) {

229 }

230 #endif // HAVE_C_MALLINFO2

231

232 - size_t netdata_mmaps = __atomic_load_n(&netdata_mmap_count, __ATOMIC_RELAXED);

232 + size_t netdata_mmaps = __atomic_load_n(&nd_mmap_count, __ATOMIC_RELAXED);

233 size_t total_mmaps = netdata_mmaps + glibc_mmaps;

234 {

235 static RRDSET *st_maps = NULL;

src/database/engine/journalfile.c

+7 -7

@@ -196,7 +196,7 @@ static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengin

196 spinlock_lock(&journalfile->mmap.spinlock);

197

198 if(!journalfile->mmap.data) {

199 - journalfile->mmap.data = mmap(NULL, journalfile->mmap.size, PROT_READ, MAP_SHARED, journalfile->mmap.fd, 0);

199 + journalfile->mmap.data = nd_mmap(NULL, journalfile->mmap.size, PROT_READ, MAP_SHARED, journalfile->mmap.fd, 0);

200 if (journalfile->mmap.data == MAP_FAILED) {

201 internal_fatal(true, "DBENGINE: failed to re-mmap() journal file v2");

202 close(journalfile->mmap.fd);

@@ -268,7 +268,7 @@ static bool journalfile_v2_mounted_data_unmount(struct rrdengine_journalfile *jo

268

269 if(!journalfile->v2.refcount) {

270 if(journalfile->mmap.data) {

271 - if (munmap(journalfile->mmap.data, journalfile->mmap.size)) {

271 + if (nd_munmap(journalfile->mmap.data, journalfile->mmap.size)) {

272 char path[RRDENG_PATH_MAX];

273 journalfile_v2_generate_path(journalfile->datafile, path, sizeof(path));

274 netdata_log_error("DBENGINE: failed to unmap index file '%s'", path);

@@ -1087,7 +1087,7 @@ int journalfile_v2_load(struct rrdengine_instance *ctx, struct rrdengine_journal

1087 }

1088

1089 usec_t mmap_start_ut = now_monotonic_usec();

1090 - uint8_t *data_start = mmap(NULL, journal_v2_file_size, PROT_READ, MAP_SHARED, fd, 0);

1090 + uint8_t *data_start = nd_mmap(NULL, journal_v2_file_size, PROT_READ, MAP_SHARED, fd, 0);

1091 if (data_start == MAP_FAILED) {

1092 close(fd);

1093 return 1;

@@ -1105,7 +1105,7 @@ int journalfile_v2_load(struct rrdengine_instance *ctx, struct rrdengine_journal

1105 else

1106 error_report("File %s is invalid and it will be rebuilt", path_v2);

1107

1108 - if (unlikely(munmap(data_start, journal_v2_file_size)))

1108 + if (unlikely(nd_munmap(data_start, journal_v2_file_size)))

1109 netdata_log_error("DBENGINE: failed to unmap '%s'", path_v2);

1110

1111 close(fd);

@@ -1116,7 +1116,7 @@ int journalfile_v2_load(struct rrdengine_instance *ctx, struct rrdengine_journal

1116 uint32_t entries = j2_header->metric_count;

1117

1118 if (unlikely(!entries)) {

1119 - if (unlikely(munmap(data_start, journal_v2_file_size)))

1119 + if (unlikely(nd_munmap(data_start, journal_v2_file_size)))

1120 netdata_log_error("DBENGINE: failed to unmap '%s'", path_v2);

1121

1122 close(fd);

@@ -1338,7 +1338,7 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno

1338 total_file_size += sizeof(struct journal_v2_block_trailer);

1339

1340 int fd_v2;

1341 - uint8_t *data_start = netdata_mmap(path, total_file_size, MAP_SHARED, 0, false, true, &fd_v2);

1341 + uint8_t *data_start = nd_mmap_advanced(path, total_file_size, MAP_SHARED, 0, false, true, &fd_v2);

1342 uint8_t *data = data_start;

1343

1344 memset(data_start, 0, extent_offset);

@@ -1494,7 +1494,7 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno

1494 resize_file_to = sizeof(j2_header);

1495 }

1496

1497 - netdata_munmap(data_start, total_file_size);

1497 + nd_munmap(data_start, total_file_size);

1498 freez(uuid_list);

1499

1500 if (likely(resize_file_to == total_file_size))

src/database/engine/rrdengineapi.c

+3 -1

@@ -1399,7 +1399,9 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {

1399 }

1400

1401 // stats.sizeof_metric = 0;

1402 - stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile));

1402 + stats.sizeof_datafile =

1403 + natural_alignment(sizeof(struct rrdengine_datafile)) +

1404 + natural_alignment(sizeof(struct rrdengine_journalfile));

1405 stats.sizeof_page_in_cache = 0; // struct_natural_alignment(sizeof(struct page_cache_descr));

1406 stats.sizeof_point_data = page_type_size[ctx->config.page_type];

1407 stats.sizeof_page_data = tier_page_size[ctx->config.tier];

src/database/rrddim.c

+2 -2

@@ -62,7 +62,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

62 size_t entries = st->db.entries;

63 if(!entries) entries = 5;

65 - rd->db.data = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, true, NULL);

65 + rd->db.data = nd_mmap_advanced(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, true, NULL);

66 if(rd->db.data) {

67 rd->db.memsize = entries * sizeof(storage_number);

68 pulse_db_rrd_memory_add(rd->db.memsize);

@@ -228,7 +228,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v

228 pulse_db_rrd_memory_sub(rd->db.memsize);

229

230 if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)

231 - netdata_munmap(rd->db.data, rd->db.memsize);

231 + nd_munmap(rd->db.data, rd->db.memsize);

232 else

233 freez(rd->db.data);

234 }

src/libnetdata/aral/aral.c

+7 -24

@@ -11,12 +11,6 @@

11 #define TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS

12 #endif

14 -#if ENV32BIT

15 -#define SYSTEM_REQUIRED_ALIGNMENT (sizeof(uintptr_t) * 2)

16 -#else

17 -#define SYSTEM_REQUIRED_ALIGNMENT (alignof(uintptr_t))

18 -#endif

19 -

14 // max mapped file size

15 #define ARAL_MAX_PAGE_SIZE_MMAP (1ULL * 1024 * 1024 * 1024)

@@ -437,19 +431,6 @@ static inline void aral_free_validate_internal_check(ARAL *ar, ARAL_FREE *fr) {

431 // --------------------------------------------------------------------------------------------------------------------

432 // page size management

433

440 -static inline size_t memory_alignment(size_t size, size_t alignment) {

441 - // return (size + alignment - 1) & ~(alignment - 1); // assumees alignment is power of 2

442 - return ((size + alignment - 1) / alignment) * alignment;

443 -}

444 -

445 -static size_t aral_get_system_page_size(void) {

446 - long int page_size = sysconf(_SC_PAGE_SIZE);

447 - if (unlikely(page_size <= 4096))

448 - return 4096;

449 - else

450 - return page_size;

451 -}

452 -

434 static size_t aral_element_slot_size(size_t requested_element_size, bool usable) {

435 // we need to add a page pointer after the element

436 // so, first align the element size to the pointer size

@@ -525,7 +506,8 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_

506 page->filename = strdupz(filename);

507 page->mapped = true;

508

528 - page->data = netdata_mmap(page->filename, size, MAP_SHARED, 0, false, ar->config.options & ARAL_DONT_DUMP, NULL);

509 + page->data =

510 + nd_mmap_advanced(page->filename, size, MAP_SHARED, 0, false, ar->config.options & ARAL_DONT_DUMP, NULL);

511 if (unlikely(!page->data))

512 fatal("ARAL: '%s' cannot allocate aral buffer of size %zu on filename '%s'",

513 ar->config.name, size, page->filename);

@@ -547,7 +529,8 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_

529

530 if (size >= ARAL_MALLOC_USE_MMAP_ABOVE) {

531 bool mapped;

550 - uint8_t *ptr = netdata_mmap(NULL, size, MAP_PRIVATE, 1, false, ar->config.options & ARAL_DONT_DUMP, NULL);

532 + uint8_t *ptr =

533 + nd_mmap_advanced(NULL, size, MAP_ANONYMOUS | MAP_PRIVATE, 1, false, ar->config.options & ARAL_DONT_DUMP, NULL);

534 if (ptr) {

535 mapped = true;

536 stats = &ar->stats->mmap;

@@ -614,7 +597,7 @@ void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_

597 stats = &ar->stats->mmap;

598 total_size = size + sizeof(ARAL_PAGE);

599

617 - netdata_munmap(page->data, page->size);

600 + nd_munmap(page->data, page->size);

601

602 if (unlikely(unlink(page->filename) == 1))

603 netdata_log_error("Cannot delete file '%s'", page->filename);

@@ -632,7 +615,7 @@ void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_

615 #else

616 if(page->mapped) {

617 stats = &ar->stats->mmap;

635 - netdata_munmap(page, page->size);

618 + nd_munmap(page, page->size);

619 }

620 else {

621 stats = &ar->stats->malloc;

@@ -1093,7 +1076,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele

1076 // ----------------------------------------------------------------------------------------------------------------

1077 // calculate allocation sizes

1078

1096 - ar->config.system_page_size = aral_get_system_page_size();

1079 + ar->config.system_page_size = os_get_system_page_size();

1080

1081 if (ar->config.initial_page_elements < 2)

1082 ar->config.initial_page_elements = 2;

src/libnetdata/libnetdata.c

-645

@@ -6,473 +6,10 @@

6 #define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED)

7 #define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED)

9 -#if !defined(MADV_DONTFORK)

10 -#define MADV_DONTFORK 0

11 -#endif

12 -

13 -#if !defined(O_NOATIME)

14 -#define O_NOATIME 0

15 -#endif

16 -

9 struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };

19 -#if defined(MADV_MERGEABLE)

20 -int enable_ksm = CONFIG_BOOLEAN_AUTO;

21 -#else

22 -int enable_ksm = 0;

23 -#endif

24 -

11 volatile sig_atomic_t netdata_exit = 0;

27 -// ----------------------------------------------------------------------------

28 -// memory allocation functions that handle failures

29 -

30 -// although netdata does not use memory allocations too often (netdata tries to

31 -// maintain its memory footprint stable during runtime, i.e. all buffers are

32 -// allocated during initialization and are adapted to current use throughout

33 -// its lifetime), these can be used to override the default system allocation

34 -// routines.

35 -

36 -#ifdef NETDATA_TRACE_ALLOCATIONS

37 -#warning NETDATA_TRACE_ALLOCATIONS ENABLED

38 -#include "Judy.h"

39 -

40 -#if defined(HAVE_DLSYM) && defined(ENABLE_DLSYM)

41 -#include <dlfcn.h>

42 -

43 -typedef void (*libc_function_t)(void);

44 -

45 -static void *malloc_first_run(size_t size);

46 -static void *(*libc_malloc)(size_t) = malloc_first_run;

47 -

48 -static void *calloc_first_run(size_t n, size_t size);

49 -static void *(*libc_calloc)(size_t, size_t) = calloc_first_run;

50 -

51 -static void *realloc_first_run(void *ptr, size_t size);

52 -static void *(*libc_realloc)(void *, size_t) = realloc_first_run;

53 -

54 -static void free_first_run(void *ptr);

55 -static void (*libc_free)(void *) = free_first_run;

56 -

57 -static char *strdup_first_run(const char *s);

58 -static char *(*libc_strdup)(const char *) = strdup_first_run;

59 -

60 -static char *strndup_first_run(const char *s, size_t len);

61 -static char *(*libc_strndup)(const char *, size_t) = strndup_first_run;

62 -

63 -static size_t malloc_usable_size_first_run(void *ptr);

64 -#ifdef HAVE_MALLOC_USABLE_SIZE

65 -static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size_first_run;

66 -#else

67 -static size_t (*libc_malloc_usable_size)(void *) = NULL;

68 -#endif

69 -

70 -static void link_system_library_function(libc_function_t *func_pptr, const char *name, bool required) {

71 - *func_pptr = dlsym(RTLD_NEXT, name);

72 - if(!*func_pptr && required) {

73 - fprintf(stderr, "FATAL: Cannot find system's %s() function.\n", name);

74 - abort();

75 - }

76 -}

77 -

78 -static void *malloc_first_run(size_t size) {

79 - link_system_library_function((libc_function_t *) &libc_malloc, "malloc", true);

80 - return libc_malloc(size);

81 -}

82 -

83 -static void *calloc_first_run(size_t n, size_t size) {

84 - link_system_library_function((libc_function_t *) &libc_calloc, "calloc", true);

85 - return libc_calloc(n, size);

86 -}

87 -

88 -static void *realloc_first_run(void *ptr, size_t size) {

89 - link_system_library_function((libc_function_t *) &libc_realloc, "realloc", true);

90 - return libc_realloc(ptr, size);

91 -}

92 -

93 -static void free_first_run(void *ptr) {

94 - link_system_library_function((libc_function_t *) &libc_free, "free", true);

95 - libc_free(ptr);

96 -}

97 -

98 -static char *strdup_first_run(const char *s) {

99 - link_system_library_function((libc_function_t *) &libc_strdup, "strdup", true);

100 - return libc_strdup(s);

101 -}

102 -

103 -static char *strndup_first_run(const char *s, size_t len) {

104 - link_system_library_function((libc_function_t *) &libc_strndup, "strndup", true);

105 - return libc_strndup(s, len);

106 -}

107 -

108 -static size_t malloc_usable_size_first_run(void *ptr) {

109 - link_system_library_function((libc_function_t *) &libc_malloc_usable_size, "malloc_usable_size", false);

110 -

111 - if(libc_malloc_usable_size)

112 - return libc_malloc_usable_size(ptr);

113 - else

114 - return 0;

115 -}

116 -

117 -void *malloc(size_t size) {

118 - return mallocz(size);

119 -}

120 -

121 -void *calloc(size_t n, size_t size) {

122 - return callocz(n, size);

123 -}

124 -

125 -void *realloc(void *ptr, size_t size) {

126 - return reallocz(ptr, size);

127 -}

128 -

129 -void *reallocarray(void *ptr, size_t n, size_t size) {

130 - return reallocz(ptr, n * size);

131 -}

132 -

133 -void free(void *ptr) {

134 - freez(ptr);

135 -}

136 -

137 -char *strdup(const char *s) {

138 - return strdupz(s);

139 -}

140 -

141 -char *strndup(const char *s, size_t len) {

142 - return strndupz(s, len);

143 -}

144 -

145 -size_t malloc_usable_size(void *ptr) {

146 - return mallocz_usable_size(ptr);

147 -}

148 -#else // !HAVE_DLSYM

149 -

150 -static void *(*libc_malloc)(size_t) = malloc;

151 -static void *(*libc_calloc)(size_t, size_t) = calloc;

152 -static void *(*libc_realloc)(void *, size_t) = realloc;

153 -static void (*libc_free)(void *) = free;

154 -

155 -#ifdef HAVE_MALLOC_USABLE_SIZE

156 -static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size;

157 -#else

158 -static size_t (*libc_malloc_usable_size)(void *) = NULL;

159 -#endif

160 -

161 -#endif // HAVE_DLSYM

162 -

163 -

164 -void posix_memfree(void *ptr) {

165 - libc_free(ptr);

166 -}

167 -

168 -struct malloc_header_signature {

169 - uint32_t magic;

170 - uint32_t size;

171 - struct malloc_trace *trace;

172 -};

173 -

174 -struct malloc_header {

175 - struct malloc_header_signature signature;

176 - uint8_t padding[(sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) ? MALLOC_ALIGNMENT - (sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) : 0];

177 - uint8_t data[];

178 -};

179 -

180 -static size_t malloc_header_size = sizeof(struct malloc_header);

181 -

182 -int malloc_trace_compare(void *A, void *B) {

183 - struct malloc_trace *a = A;

184 - struct malloc_trace *b = B;

185 - return strcmp(a->function, b->function);

186 -}

187 -

188 -static avl_tree_lock malloc_trace_index = {

189 - .avl_tree = {

190 - .root = NULL,

191 - .compar = malloc_trace_compare},

192 - .rwlock = AVL_LOCK_INITIALIZER

193 -};

194 -

195 -int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data) {

196 - return avl_traverse_lock(&malloc_trace_index, callback, data);

197 -}

198 -

199 -NEVERNULL WARNUNUSED

200 -static struct malloc_trace *malloc_trace_find_or_create(const char *file, const char *function, size_t line) {

201 - struct malloc_trace tmp = {

202 - .line = line,

203 - .function = function,

204 - .file = file,

205 - };

206 -

207 - struct malloc_trace *t = (struct malloc_trace *)avl_search_lock(&malloc_trace_index, (avl_t *)&tmp);

208 - if(!t) {

209 - t = libc_calloc(1, sizeof(struct malloc_trace));

210 - if(!t) fatal("No memory");

211 - t->line = line;

212 - t->function = function;

213 - t->file = file;

214 -

215 - struct malloc_trace *t2 = (struct malloc_trace *)avl_insert_lock(&malloc_trace_index, (avl_t *)t);

216 - if(t2 != t)

217 - free(t);

218 -

219 - t = t2;

220 - }

221 -

222 - if(!t)

223 - fatal("Cannot insert to AVL");

224 -

225 - return t;

226 -}

227 -

228 -void malloc_trace_mmap(size_t size) {

229 - struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);

230 - size_t_atomic_count(add, p->mmap_calls, 1);

231 - size_t_atomic_count(add, p->allocations, 1);

232 - size_t_atomic_bytes(add, p->bytes, size);

233 -}

234 -

235 -void malloc_trace_munmap(size_t size) {

236 - struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);

237 - size_t_atomic_count(add, p->munmap_calls, 1);

238 - size_t_atomic_count(sub, p->allocations, 1);

239 - size_t_atomic_bytes(sub, p->bytes, size);

240 -}

241 -

242 -void *mallocz_int(size_t size, const char *file, const char *function, size_t line) {

243 - struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

244 -

245 - size_t_atomic_count(add, p->malloc_calls, 1);

246 - size_t_atomic_count(add, p->allocations, 1);

247 - size_t_atomic_bytes(add, p->bytes, size);

248 -

249 - struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);

250 - if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

251 - t->signature.magic = 0x0BADCAFE;

252 - t->signature.trace = p;

253 - t->signature.size = size;

254 -

255 -#ifdef NETDATA_INTERNAL_CHECKS

256 - for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

257 - t->padding[i] = 0xFF;

258 -#endif

259 -

260 - return (void *)&t->data;

261 -}

262 -

263 -void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line) {

264 - struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

265 - size = nmemb * size;

266 -

267 - size_t_atomic_count(add, p->calloc_calls, 1);

268 - size_t_atomic_count(add, p->allocations, 1);

269 - size_t_atomic_bytes(add, p->bytes, size);

270 -

271 - struct malloc_header *t = (struct malloc_header *)libc_calloc(1, malloc_header_size + size);

272 - if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

273 - t->signature.magic = 0x0BADCAFE;

274 - t->signature.trace = p;

275 - t->signature.size = size;

276 -

277 -#ifdef NETDATA_INTERNAL_CHECKS

278 - for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

279 - t->padding[i] = 0xFF;

280 -#endif

281 -

282 - return &t->data;

283 -}

284 -

285 -char *strdupz_int(const char *s, const char *file, const char *function, size_t line) {

286 - struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

287 - size_t size = strlen(s) + 1;

288 -

289 - size_t_atomic_count(add, p->strdup_calls, 1);

290 - size_t_atomic_count(add, p->allocations, 1);

291 - size_t_atomic_bytes(add, p->bytes, size);

292 -

293 - struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);

294 - if (unlikely(!t)) fatal("strdupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

295 - t->signature.magic = 0x0BADCAFE;

296 - t->signature.trace = p;

297 - t->signature.size = size;

298 -

299 -#ifdef NETDATA_INTERNAL_CHECKS

300 - for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

301 - t->padding[i] = 0xFF;

302 -#endif

303 -

304 - memcpy(&t->data, s, size);

305 - return (char *)&t->data;

306 -}

307 -

308 -char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line) {

309 - struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

310 - size_t size = len + 1;

311 -

312 - size_t_atomic_count(add, p->strdup_calls, 1);

313 - size_t_atomic_count(add, p->allocations, 1);

314 - size_t_atomic_bytes(add, p->bytes, size);

315 -

316 - struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);

317 - if (unlikely(!t)) fatal("strndupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

318 - t->signature.magic = 0x0BADCAFE;

319 - t->signature.trace = p;

320 - t->signature.size = size;

321 -

322 -#ifdef NETDATA_INTERNAL_CHECKS

323 - for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

324 - t->padding[i] = 0xFF;

325 -#endif

326 -

327 - memcpy(&t->data, s, size);

328 - t->data[len] = '\0';

329 - return (char *)&t->data;

330 -}

331 -

332 -static struct malloc_header *malloc_get_header(void *ptr, const char *caller, const char *file, const char *function, size_t line) {

333 - uint8_t *ret = (uint8_t *)ptr - malloc_header_size;

334 - struct malloc_header *t = (struct malloc_header *)ret;

335 -

336 - if(t->signature.magic != 0x0BADCAFE) {

337 - netdata_log_error("pointer %p is not our pointer (called %s() from %zu@%s, %s()).", ptr, caller, line, file, function);

338 - return NULL;

339 - }

340 -

341 - return t;

342 -}

343 -

344 -void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line) {

345 - if(!ptr) return mallocz_int(size, file, function, line);

346 -

347 - struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);

348 - if(!t)

349 - return libc_realloc(ptr, size);

350 -

351 - if(t->signature.size == size) return ptr;

352 - size_t_atomic_count(add, t->signature.trace->free_calls, 1);

353 - size_t_atomic_count(sub, t->signature.trace->allocations, 1);

354 - size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size);

355 -

356 - struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

357 - size_t_atomic_count(add, p->realloc_calls, 1);

358 - size_t_atomic_count(add, p->allocations, 1);

359 - size_t_atomic_bytes(add, p->bytes, size);

360 -

361 - t = (struct malloc_header *)libc_realloc(t, malloc_header_size + size);

362 - if (unlikely(!t)) fatal("reallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

363 - t->signature.magic = 0x0BADCAFE;

364 - t->signature.trace = p;

365 - t->signature.size = size;

366 -

367 -#ifdef NETDATA_INTERNAL_CHECKS

368 - for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

369 - t->padding[i] = 0xFF;

370 -#endif

371 -

372 - return (void *)&t->data;

373 -}

374 -

375 -size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line) {

376 - if(unlikely(!ptr)) return 0;

377 -

378 - struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);

379 - if(!t) {

380 - if(libc_malloc_usable_size)

381 - return libc_malloc_usable_size(ptr);

382 - else

383 - return 0;

384 - }

385 -

386 - return t->signature.size;

387 -}

388 -

389 -void freez_int(void *ptr, const char *file, const char *function, size_t line) {

390 - if(unlikely(!ptr)) return;

391 -

392 - struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);

393 - if(!t) {

394 - libc_free(ptr);

395 - return;

396 - }

397 -

398 - size_t_atomic_count(add, t->signature.trace->free_calls, 1);

399 - size_t_atomic_count(sub, t->signature.trace->allocations, 1);

400 - size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size);

401 -

402 -#ifdef NETDATA_INTERNAL_CHECKS

403 - // it should crash if it is used after freeing it

404 - memset(t, 0, malloc_header_size + t->signature.size);

405 -#endif

406 -

407 - libc_free(t);

408 -}

409 -#else

410 -

411 -char *strdupz(const char *s) {

412 - char *t = strdup(s);

413 - if (unlikely(!t)) {

414 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

415 - fatal("Cannot strdup() string '%s' (system memory available bytes: %lu)", s, sm.ram_available_bytes);

416 - }

417 - return t;

418 -}

419 -

420 -char *strndupz(const char *s, size_t len) {

421 - char *t = strndup(s, len);

422 - if (unlikely(!t)) {

423 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

424 - fatal("Cannot strndup() string '%s' of len %zu (system memory available bytes: %lu)", s, len, sm.ram_available_bytes);

425 - }

426 - return t;

427 -}

428 -

429 -// If ptr is NULL, no operation is performed.

430 -void freez(void *ptr) {

431 - if(likely(ptr)) free(ptr);

432 -}

433 -

434 -void *mallocz(size_t size) {

435 - void *p = malloc(size);

436 - if (unlikely(!p)) {

437 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

438 - fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", size, sm.ram_available_bytes);

439 - }

440 - return p;

441 -}

442 -

443 -void *callocz(size_t nmemb, size_t size) {

444 - void *p = calloc(nmemb, size);

445 - if (unlikely(!p)) {

446 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

447 - fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", nmemb * size, sm.ram_available_bytes);

448 - }

449 - return p;

450 -}

451 -

452 -void *reallocz(void *ptr, size_t size) {

453 - void *p = realloc(ptr, size);

454 - if (unlikely(!p)) {

455 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

456 - fatal("Cannot re-allocate memory to %zu bytes. (system memory available bytes: %lu)", size, sm.ram_available_bytes);

457 - }

458 - return p;

459 -}

460 -

461 -void posix_memfree(void *ptr) {

462 - free(ptr);

463 -}

464 -#endif

465 -

466 -void mallocz_release_as_much_memory_to_the_system(void) {

467 -#if defined(HAVE_C_MALLOC_TRIM)

468 - static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

469 - if(spinlock_trylock(&spinlock)) {

470 - malloc_trim(0);

471 - spinlock_unlock(&spinlock);

472 - }

473 -#endif

474 -}

475 -

13 // --------------------------------------------------------------------------------------------------------------------

15 void json_escape_string(char *dst, const char *src, size_t size) {

@@ -506,188 +43,6 @@ void json_fix_string(char *s) {

43 }

44 }

509 -static int memory_file_open(const char *filename, size_t size) {

510 - // netdata_log_info("memory_file_open('%s', %zu", filename, size);

511 -

512 - int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME | O_CLOEXEC, 0664);

513 - if (fd != -1) {

514 - if (lseek(fd, size, SEEK_SET) == (off_t) size) {

515 - if (write(fd, "", 1) == 1) {

516 - if (ftruncate(fd, size))

517 - netdata_log_error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size);

518 - }

519 - else

520 - netdata_log_error("Cannot write to file '%s' at position %zu.", filename, size);

521 - }

522 - else

523 - netdata_log_error("Cannot seek file '%s' to size %zu.", filename, size);

524 - }

525 - else

526 - netdata_log_error("Cannot create/open file '%s'.", filename);

527 -

528 - return fd;

529 -}

530 -

531 -inline int madvise_sequential(void *mem, size_t len) {

532 - static int logger = 1;

533 - int ret = madvise(mem, len, MADV_SEQUENTIAL);

534 -

535 - if (ret != 0 && logger-- > 0)

536 - netdata_log_error("madvise(MADV_SEQUENTIAL) of size %zu, failed.", len);

537 - return ret;

538 -}

539 -

540 -inline int madvise_random(void *mem, size_t len) {

541 - static int logger = 1;

542 - int ret = madvise(mem, len, MADV_RANDOM);

543 -

544 - if (ret != 0 && logger-- > 0)

545 - netdata_log_error("madvise(MADV_RANDOM) of size %zu, failed.", len);

546 - return ret;

547 -}

548 -

549 -inline int madvise_dontfork(void *mem, size_t len) {

550 - static int logger = 1;

551 - int ret = madvise(mem, len, MADV_DONTFORK);

552 -

553 - if (ret != 0 && logger-- > 0)

554 - netdata_log_error("madvise(MADV_DONTFORK) of size %zu, failed.", len);

555 - return ret;

556 -}

557 -

558 -inline int madvise_willneed(void *mem, size_t len) {

559 - static int logger = 1;

560 - int ret = madvise(mem, len, MADV_WILLNEED);

561 -

562 - if (ret != 0 && logger-- > 0)

563 - netdata_log_error("madvise(MADV_WILLNEED) of size %zu, failed.", len);

564 - return ret;

565 -}

566 -

567 -inline int madvise_dontneed(void *mem, size_t len) {

568 - static int logger = 1;

569 - int ret = madvise(mem, len, MADV_DONTNEED);

570 -

571 - if (ret != 0 && logger-- > 0)

572 - netdata_log_error("madvise(MADV_DONTNEED) of size %zu, failed.", len);

573 - return ret;

574 -}

575 -

576 -inline int madvise_dontdump(void *mem __maybe_unused, size_t len __maybe_unused) {

577 -#if __linux__

578 - static int logger = 1;

579 - int ret = madvise(mem, len, MADV_DONTDUMP);

580 -

581 - if (ret != 0 && logger-- > 0)

582 - netdata_log_error("madvise(MADV_DONTDUMP) of size %zu, failed.", len);

583 - return ret;

584 -#else

585 - return 0;

586 -#endif

587 -}

588 -

589 -inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused) {

590 -#ifdef MADV_MERGEABLE

591 - static int logger = 1;

592 - int ret = madvise(mem, len, MADV_MERGEABLE);

593 -

594 - if (ret != 0 && logger-- > 0)

595 - netdata_log_error("madvise(MADV_MERGEABLE) of size %zu, failed.", len);

596 - return ret;

597 -#else

598 - return 0;

599 -#endif

600 -}

601 -

602 -size_t netdata_mmap_count = 0;

603 -

604 -void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only, bool dont_dump, int *open_fd)

605 -{

606 - // netdata_log_info("netdata_mmap('%s', %zu", filename, size);

607 -

608 - // MAP_SHARED is used in memory mode map

609 - // MAP_PRIVATE is used in memory mode ram and save

610 -

611 - if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE)))

612 - fatal("Neither MAP_SHARED or MAP_PRIVATE were given to netdata_mmap()");

613 -

614 - if(unlikely((flags & MAP_SHARED) && (flags & MAP_PRIVATE)))

615 - fatal("Both MAP_SHARED and MAP_PRIVATE were given to netdata_mmap()");

616 -

617 - if(unlikely((flags & MAP_SHARED) && (!filename || !*filename)))

618 - fatal("MAP_SHARED requested, without a filename to netdata_mmap()");

619 -

620 - // don't enable ksm is the global setting is disabled

621 - if(unlikely(!enable_ksm)) ksm = 0;

622 -

623 - // KSM only merges anonymous (private) pages, never pagecache (file) pages

624 - // but MAP_PRIVATE without MAP_ANONYMOUS it fails too, so we need it always

625 - if((flags & MAP_PRIVATE)) flags |= MAP_ANONYMOUS;

626 -

627 - int fd = -1;

628 - void *mem = MAP_FAILED;

629 -

630 - if(filename && *filename) {

631 - // open/create the file to be used

632 - fd = memory_file_open(filename, size);

633 - if(fd == -1) goto cleanup;

634 - }

635 -

636 - int fd_for_mmap = fd;

637 - if(fd != -1 && (flags & MAP_PRIVATE)) {

638 - // this is MAP_PRIVATE allocation

639 - // no need for mmap() to use our fd

640 - // we will copy the file into the memory allocated

641 - fd_for_mmap = -1;

642 - }

643 -

644 - mem = mmap(NULL, size, read_only ? PROT_READ : PROT_READ | PROT_WRITE, flags, fd_for_mmap, 0);

645 - if (mem != MAP_FAILED) {

646 -

647 -#ifdef NETDATA_TRACE_ALLOCATIONS

648 - malloc_trace_mmap(size);

649 -#endif

650 -

651 - // if we have a file open, but we didn't give it to mmap(),

652 - // we have to read the file into the memory block we allocated

653 - if(fd != -1 && fd_for_mmap == -1) {

654 - if (lseek(fd, 0, SEEK_SET) == 0) {

655 - if (read(fd, mem, size) != (ssize_t) size)

656 - netdata_log_info("Cannot read from file '%s'", filename);

657 - }

658 - else netdata_log_info("Cannot seek to beginning of file '%s'.", filename);

659 - }

660 -

661 - // madvise_sequential(mem, size);

662 - // madvise_dontfork(mem, size); // aral is initialized before we daemonize

663 - if(dont_dump) madvise_dontdump(mem, size);

664 - // if(flags & MAP_SHARED) madvise_willneed(mem, size);

665 - if(ksm) madvise_mergeable(mem, size);

666 - }

667 -

668 -cleanup:

669 - if(fd != -1) {

670 - if (open_fd)

671 - *open_fd = fd;

672 - else

673 - close(fd);

674 - }

675 - if(mem == MAP_FAILED) return NULL;

676 - __atomic_add_fetch(&netdata_mmap_count, 1, __ATOMIC_RELAXED);

677 - errno_clear();

678 - return mem;

679 -}

680 -

681 -int netdata_munmap(void *ptr, size_t size) {

682 -#ifdef NETDATA_TRACE_ALLOCATIONS

683 - malloc_trace_munmap(size);

684 -#endif

685 - int rc = munmap(ptr, size);

686 - if(rc == 0)

687 - __atomic_sub_fetch(&netdata_mmap_count, 1, __ATOMIC_RELAXED);

688 - return rc;

689 -}

690 -

46 char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {

47 char *s = fgets(buf, (int)buf_size, fp);

48 if (!s) return NULL;

src/libnetdata/libnetdata.h

+3 -83

@@ -8,6 +8,9 @@ extern "C" {

8 # endif

10 #include "common.h"

11 +#include "memory/alignment.h"

12 +#include "memory/nd-mallocz.h"

13 +#include "memory/nd-mmap.h"

14 #include "log/nd_log-fatal.h"

15 #include "atomics/atomics.h"

@@ -23,61 +26,15 @@ extern "C" {

26 #include "storage-point.h"

27 #include "paths/paths.h"

26 -int madvise_sequential(void *mem, size_t len);

27 -int madvise_random(void *mem, size_t len);

28 -int madvise_dontfork(void *mem, size_t len);

29 -int madvise_willneed(void *mem, size_t len);

30 -int madvise_dontneed(void *mem, size_t len);

31 -int madvise_dontdump(void *mem, size_t len);

32 -int madvise_mergeable(void *mem, size_t len);

33 -

29 int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args);

30 int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4);

37 -// memory allocation functions that handle failures

38 -#ifdef NETDATA_TRACE_ALLOCATIONS

39 -int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data);

40 -

41 -#define strdupz(s) strdupz_int(s, __FILE__, __FUNCTION__, __LINE__)

42 -#define strndupz(s, len) strndupz_int(s, len, __FILE__, __FUNCTION__, __LINE__)

43 -#define callocz(nmemb, size) callocz_int(nmemb, size, __FILE__, __FUNCTION__, __LINE__)

44 -#define mallocz(size) mallocz_int(size, __FILE__, __FUNCTION__, __LINE__)

45 -#define reallocz(ptr, size) reallocz_int(ptr, size, __FILE__, __FUNCTION__, __LINE__)

46 -#define freez(ptr) freez_int(ptr, __FILE__, __FUNCTION__, __LINE__)

47 -#define mallocz_usable_size(ptr) mallocz_usable_size_int(ptr, __FILE__, __FUNCTION__, __LINE__)

48 -

49 -char *strdupz_int(const char *s, const char *file, const char *function, size_t line);

50 -char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line);

51 -void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line);

52 -void *mallocz_int(size_t size, const char *file, const char *function, size_t line);

53 -void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line);

54 -void freez_int(void *ptr, const char *file, const char *function, size_t line);

55 -size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line);

56 -

57 -#else // NETDATA_TRACE_ALLOCATIONS

58 -char *strdupz(const char *s) MALLOCLIKE NEVERNULL;

59 -char *strndupz(const char *s, size_t len) MALLOCLIKE NEVERNULL;

60 -void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;

61 -void *mallocz(size_t size) MALLOCLIKE NEVERNULL;

62 -void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL;

63 -void freez(void *ptr);

64 -#endif // NETDATA_TRACE_ALLOCATIONS

65 -

66 -void mallocz_release_as_much_memory_to_the_system(void);

67 -void posix_memfree(void *ptr);

68 -

32 void json_escape_string(char *dst, const char *src, size_t size);

33 void json_fix_string(char *s);

72 -extern size_t netdata_mmap_count;

73 -void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only, bool dont_dump, int *open_fd);

74 -int netdata_munmap(void *ptr, size_t size);

75 -int memory_file_save(const char *filename, void *mem, size_t size);

36 extern struct rlimit rlimit_nofile;

79 -extern int enable_ksm;

80 -

38 char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);

40 int verify_netdata_host_prefix(bool log_msg);

@@ -188,43 +145,6 @@ extern const char *netdata_configured_host_prefix;

145 #include "functions_evloop/functions_evloop.h"

146 #include "query_progress/progress.h"

147

191 -static inline size_t struct_natural_alignment(size_t size) __attribute__((const));

192 -

193 -#define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)

194 -static inline size_t struct_natural_alignment(size_t size) {

195 - if(unlikely(size % STRUCT_NATURAL_ALIGNMENT))

196 - size = size + STRUCT_NATURAL_ALIGNMENT - (size % STRUCT_NATURAL_ALIGNMENT);

197 -

198 - return size;

199 -}

200 -

201 -#ifdef NETDATA_TRACE_ALLOCATIONS

202 -struct malloc_trace {

203 - avl_t avl;

204 -

205 - const char *function;

206 - const char *file;

207 - size_t line;

208 -

209 - size_t malloc_calls;

210 - size_t calloc_calls;

211 - size_t realloc_calls;

212 - size_t strdup_calls;

213 - size_t free_calls;

214 -

215 - size_t mmap_calls;

216 - size_t munmap_calls;

217 -

218 - size_t allocations;

219 - size_t bytes;

220 -

221 - struct rrddim *rd_bytes;

222 - struct rrddim *rd_allocations;

223 - struct rrddim *rd_avg_alloc;

224 - struct rrddim *rd_ops;

225 -};

226 -#endif // NETDATA_TRACE_ALLOCATIONS

227 -

148 static inline PPvoid_t JudyLFirstThenNext(Pcvoid_t PArray, Word_t * PIndex, bool *first) {

149 if(unlikely(*first)) {

150 *first = false;

src/libnetdata/memory/alignment.h new

+25

@@ -0,0 +1,25 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_ALIGNMENT_H

4 +#define NETDATA_ALIGNMENT_H

5 +

6 +#include "../common.h"

7 +

8 +#if ENV32BIT

9 +#define SYSTEM_REQUIRED_ALIGNMENT (sizeof(uintptr_t) * 2)

10 +#else

11 +#define SYSTEM_REQUIRED_ALIGNMENT (alignof(uintptr_t))

12 +#endif

13 +

14 +static inline size_t memory_alignment(size_t size, size_t alignment) __attribute__((const));

15 +static inline size_t memory_alignment(size_t size, size_t alignment) {

16 + // return (size + alignment - 1) & ~(alignment - 1); // assumes alignment is power of 2

17 + return ((size + alignment - 1) / alignment) * alignment;

18 +}

19 +

20 +static inline size_t natural_alignment(size_t size) __attribute__((const));

21 +static inline size_t natural_alignment(size_t size) {

22 + return memory_alignment(size, SYSTEM_REQUIRED_ALIGNMENT);

23 +}

24 +

25 +#endif //NETDATA_ALIGNMENT_H

src/libnetdata/memory/nd-mallocz.c new

+452

@@ -0,0 +1,452 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "../libnetdata.h"

4 +

5 +// ----------------------------------------------------------------------------

6 +// memory allocation functions that handle failures

7 +

8 +// although netdata does not use memory allocations too often (netdata tries to

9 +// maintain its memory footprint stable during runtime, i.e. all buffers are

10 +// allocated during initialization and are adapted to current use throughout

11 +// its lifetime), these can be used to override the default system allocation

12 +// routines.

13 +

14 +#ifdef NETDATA_TRACE_ALLOCATIONS

15 +#warning NETDATA_TRACE_ALLOCATIONS ENABLED

16 +#include "Judy.h"

17 +

18 +#if defined(HAVE_DLSYM) && defined(ENABLE_DLSYM)

19 +#include <dlfcn.h>

20 +

21 +typedef void (*libc_function_t)(void);

22 +

23 +static void *malloc_first_run(size_t size);

24 +static void *(*libc_malloc)(size_t) = malloc_first_run;

25 +

26 +static void *calloc_first_run(size_t n, size_t size);

27 +static void *(*libc_calloc)(size_t, size_t) = calloc_first_run;

28 +

29 +static void *realloc_first_run(void *ptr, size_t size);

30 +static void *(*libc_realloc)(void *, size_t) = realloc_first_run;

31 +

32 +static void free_first_run(void *ptr);

33 +static void (*libc_free)(void *) = free_first_run;

34 +

35 +static char *strdup_first_run(const char *s);

36 +static char *(*libc_strdup)(const char *) = strdup_first_run;

37 +

38 +static char *strndup_first_run(const char *s, size_t len);

39 +static char *(*libc_strndup)(const char *, size_t) = strndup_first_run;

40 +

41 +static size_t malloc_usable_size_first_run(void *ptr);

42 +#ifdef HAVE_MALLOC_USABLE_SIZE

43 +static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size_first_run;

44 +#else

45 +static size_t (*libc_malloc_usable_size)(void *) = NULL;

46 +#endif

47 +

48 +static void link_system_library_function(libc_function_t *func_pptr, const char *name, bool required) {

49 + *func_pptr = dlsym(RTLD_NEXT, name);

50 + if(!*func_pptr && required) {

51 + fprintf(stderr, "FATAL: Cannot find system's %s() function.\n", name);

52 + abort();

53 + }

54 +}

55 +

56 +static void *malloc_first_run(size_t size) {

57 + link_system_library_function((libc_function_t *) &libc_malloc, "malloc", true);

58 + return libc_malloc(size);

59 +}

60 +

61 +static void *calloc_first_run(size_t n, size_t size) {

62 + link_system_library_function((libc_function_t *) &libc_calloc, "calloc", true);

63 + return libc_calloc(n, size);

64 +}

65 +

66 +static void *realloc_first_run(void *ptr, size_t size) {

67 + link_system_library_function((libc_function_t *) &libc_realloc, "realloc", true);

68 + return libc_realloc(ptr, size);

69 +}

70 +

71 +static void free_first_run(void *ptr) {

72 + link_system_library_function((libc_function_t *) &libc_free, "free", true);

73 + libc_free(ptr);

74 +}

75 +

76 +static char *strdup_first_run(const char *s) {

77 + link_system_library_function((libc_function_t *) &libc_strdup, "strdup", true);

78 + return libc_strdup(s);

79 +}

80 +

81 +static char *strndup_first_run(const char *s, size_t len) {

82 + link_system_library_function((libc_function_t *) &libc_strndup, "strndup", true);

83 + return libc_strndup(s, len);

84 +}

85 +

86 +static size_t malloc_usable_size_first_run(void *ptr) {

87 + link_system_library_function((libc_function_t *) &libc_malloc_usable_size, "malloc_usable_size", false);

88 +

89 + if(libc_malloc_usable_size)

90 + return libc_malloc_usable_size(ptr);

91 + else

92 + return 0;

93 +}

94 +

95 +void *malloc(size_t size) {

96 + return mallocz(size);

97 +}

98 +

99 +void *calloc(size_t n, size_t size) {

100 + return callocz(n, size);

101 +}

102 +

103 +void *realloc(void *ptr, size_t size) {

104 + return reallocz(ptr, size);

105 +}

106 +

107 +void *reallocarray(void *ptr, size_t n, size_t size) {

108 + return reallocz(ptr, n * size);

109 +}

110 +

111 +void free(void *ptr) {

112 + freez(ptr);

113 +}

114 +

115 +char *strdup(const char *s) {

116 + return strdupz(s);

117 +}

118 +

119 +char *strndup(const char *s, size_t len) {

120 + return strndupz(s, len);

121 +}

122 +

123 +size_t malloc_usable_size(void *ptr) {

124 + return mallocz_usable_size(ptr);

125 +}

126 +#else // !HAVE_DLSYM

127 +

128 +static void *(*libc_malloc)(size_t) = malloc;

129 +static void *(*libc_calloc)(size_t, size_t) = calloc;

130 +static void *(*libc_realloc)(void *, size_t) = realloc;

131 +static void (*libc_free)(void *) = free;

132 +

133 +#ifdef HAVE_MALLOC_USABLE_SIZE

134 +static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size;

135 +#else

136 +static size_t (*libc_malloc_usable_size)(void *) = NULL;

137 +#endif

138 +

139 +#endif // HAVE_DLSYM

140 +

141 +

142 +void posix_memfree(void *ptr) {

143 + libc_free(ptr);

144 +}

145 +

146 +struct malloc_header_signature {

147 + uint32_t magic;

148 + uint32_t size;

149 + struct malloc_trace *trace;

150 +};

151 +

152 +struct malloc_header {

153 + struct malloc_header_signature signature;

154 + uint8_t padding[(sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) ? MALLOC_ALIGNMENT - (sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) : 0];

155 + uint8_t data[];

156 +};

157 +

158 +static size_t malloc_header_size = sizeof(struct malloc_header);

159 +

160 +int malloc_trace_compare(void *A, void *B) {

161 + struct malloc_trace *a = A;

162 + struct malloc_trace *b = B;

163 + return strcmp(a->function, b->function);

164 +}

165 +

166 +static avl_tree_lock malloc_trace_index = {

167 + .avl_tree = {

168 + .root = NULL,

169 + .compar = malloc_trace_compare},

170 + .rwlock = AVL_LOCK_INITIALIZER

171 +};

172 +

173 +int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data) {

174 + return avl_traverse_lock(&malloc_trace_index, callback, data);

175 +}

176 +

177 +NEVERNULL WARNUNUSED

178 + static struct malloc_trace *malloc_trace_find_or_create(const char *file, const char *function, size_t line) {

179 + struct malloc_trace tmp = {

180 + .line = line,

181 + .function = function,

182 + .file = file,

183 + };

184 +

185 + struct malloc_trace *t = (struct malloc_trace *)avl_search_lock(&malloc_trace_index, (avl_t *)&tmp);

186 + if(!t) {

187 + t = libc_calloc(1, sizeof(struct malloc_trace));

188 + if(!t) fatal("No memory");

189 + t->line = line;

190 + t->function = function;

191 + t->file = file;

192 +

193 + struct malloc_trace *t2 = (struct malloc_trace *)avl_insert_lock(&malloc_trace_index, (avl_t *)t);

194 + if(t2 != t)

195 + free(t);

196 +

197 + t = t2;

198 + }

199 +

200 + if(!t)

201 + fatal("Cannot insert to AVL");

202 +

203 + return t;

204 +}

205 +

206 +void malloc_trace_mmap(size_t size) {

207 + struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);

208 + size_t_atomic_count(add, p->mmap_calls, 1);

209 + size_t_atomic_count(add, p->allocations, 1);

210 + size_t_atomic_bytes(add, p->bytes, size);

211 +}

212 +

213 +void malloc_trace_munmap(size_t size) {

214 + struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);

215 + size_t_atomic_count(add, p->munmap_calls, 1);

216 + size_t_atomic_count(sub, p->allocations, 1);

217 + size_t_atomic_bytes(sub, p->bytes, size);

218 +}

219 +

220 +void *mallocz_int(size_t size, const char *file, const char *function, size_t line) {

221 + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

222 +

223 + size_t_atomic_count(add, p->malloc_calls, 1);

224 + size_t_atomic_count(add, p->allocations, 1);

225 + size_t_atomic_bytes(add, p->bytes, size);

226 +

227 + struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);

228 + if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

229 + t->signature.magic = 0x0BADCAFE;

230 + t->signature.trace = p;

231 + t->signature.size = size;

232 +

233 +#ifdef NETDATA_INTERNAL_CHECKS

234 + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

235 + t->padding[i] = 0xFF;

236 +#endif

237 +

238 + return (void *)&t->data;

239 +}

240 +

241 +void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line) {

242 + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

243 + size = nmemb * size;

244 +

245 + size_t_atomic_count(add, p->calloc_calls, 1);

246 + size_t_atomic_count(add, p->allocations, 1);

247 + size_t_atomic_bytes(add, p->bytes, size);

248 +

249 + struct malloc_header *t = (struct malloc_header *)libc_calloc(1, malloc_header_size + size);

250 + if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

251 + t->signature.magic = 0x0BADCAFE;

252 + t->signature.trace = p;

253 + t->signature.size = size;

254 +

255 +#ifdef NETDATA_INTERNAL_CHECKS

256 + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

257 + t->padding[i] = 0xFF;

258 +#endif

259 +

260 + return &t->data;

261 +}

262 +

263 +char *strdupz_int(const char *s, const char *file, const char *function, size_t line) {

264 + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

265 + size_t size = strlen(s) + 1;

266 +

267 + size_t_atomic_count(add, p->strdup_calls, 1);

268 + size_t_atomic_count(add, p->allocations, 1);

269 + size_t_atomic_bytes(add, p->bytes, size);

270 +

271 + struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);

272 + if (unlikely(!t)) fatal("strdupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

273 + t->signature.magic = 0x0BADCAFE;

274 + t->signature.trace = p;

275 + t->signature.size = size;

276 +

277 +#ifdef NETDATA_INTERNAL_CHECKS

278 + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

279 + t->padding[i] = 0xFF;

280 +#endif

281 +

282 + memcpy(&t->data, s, size);

283 + return (char *)&t->data;

284 +}

285 +

286 +char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line) {

287 + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

288 + size_t size = len + 1;

289 +

290 + size_t_atomic_count(add, p->strdup_calls, 1);

291 + size_t_atomic_count(add, p->allocations, 1);

292 + size_t_atomic_bytes(add, p->bytes, size);

293 +

294 + struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);

295 + if (unlikely(!t)) fatal("strndupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

296 + t->signature.magic = 0x0BADCAFE;

297 + t->signature.trace = p;

298 + t->signature.size = size;

299 +

300 +#ifdef NETDATA_INTERNAL_CHECKS

301 + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

302 + t->padding[i] = 0xFF;

303 +#endif

304 +

305 + memcpy(&t->data, s, size);

306 + t->data[len] = '\0';

307 + return (char *)&t->data;

308 +}

309 +

310 +static struct malloc_header *malloc_get_header(void *ptr, const char *caller, const char *file, const char *function, size_t line) {

311 + uint8_t *ret = (uint8_t *)ptr - malloc_header_size;

312 + struct malloc_header *t = (struct malloc_header *)ret;

313 +

314 + if(t->signature.magic != 0x0BADCAFE) {

315 + netdata_log_error("pointer %p is not our pointer (called %s() from %zu@%s, %s()).", ptr, caller, line, file, function);

316 + return NULL;

317 + }

318 +

319 + return t;

320 +}

321 +

322 +void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line) {

323 + if(!ptr) return mallocz_int(size, file, function, line);

324 +

325 + struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);

326 + if(!t)

327 + return libc_realloc(ptr, size);

328 +

329 + if(t->signature.size == size) return ptr;

330 + size_t_atomic_count(add, t->signature.trace->free_calls, 1);

331 + size_t_atomic_count(sub, t->signature.trace->allocations, 1);

332 + size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size);

333 +

334 + struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);

335 + size_t_atomic_count(add, p->realloc_calls, 1);

336 + size_t_atomic_count(add, p->allocations, 1);

337 + size_t_atomic_bytes(add, p->bytes, size);

338 +

339 + t = (struct malloc_header *)libc_realloc(t, malloc_header_size + size);

340 + if (unlikely(!t)) fatal("reallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);

341 + t->signature.magic = 0x0BADCAFE;

342 + t->signature.trace = p;

343 + t->signature.size = size;

344 +

345 +#ifdef NETDATA_INTERNAL_CHECKS

346 + for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded

347 + t->padding[i] = 0xFF;

348 +#endif

349 +

350 + return (void *)&t->data;

351 +}

352 +

353 +size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line) {

354 + if(unlikely(!ptr)) return 0;

355 +

356 + struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);

357 + if(!t) {

358 + if(libc_malloc_usable_size)

359 + return libc_malloc_usable_size(ptr);

360 + else

361 + return 0;

362 + }

363 +

364 + return t->signature.size;

365 +}

366 +

367 +void freez_int(void *ptr, const char *file, const char *function, size_t line) {

368 + if(unlikely(!ptr)) return;

369 +

370 + struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);

371 + if(!t) {

372 + libc_free(ptr);

373 + return;

374 + }

375 +

376 + size_t_atomic_count(add, t->signature.trace->free_calls, 1);

377 + size_t_atomic_count(sub, t->signature.trace->allocations, 1);

378 + size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size);

379 +

380 +#ifdef NETDATA_INTERNAL_CHECKS

381 + // it should crash if it is used after freeing it

382 + memset(t, 0, malloc_header_size + t->signature.size);

383 +#endif

384 +

385 + libc_free(t);

386 +}

387 +#else

388 +

389 +char *strdupz(const char *s) {

390 + char *t = strdup(s);

391 + if (unlikely(!t)) {

392 + OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

393 + fatal("Cannot strdup() string '%s' (system memory available bytes: %lu)", s, sm.ram_available_bytes);

394 + }

395 + return t;

396 +}

397 +

398 +char *strndupz(const char *s, size_t len) {

399 + char *t = strndup(s, len);

400 + if (unlikely(!t)) {

401 + OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

402 + fatal("Cannot strndup() string '%s' of len %zu (system memory available bytes: %lu)", s, len, sm.ram_available_bytes);

403 + }

404 + return t;

405 +}

406 +

407 +// If ptr is NULL, no operation is performed.

408 +void freez(void *ptr) {

409 + if(likely(ptr)) free(ptr);

410 +}

411 +

412 +void *mallocz(size_t size) {

413 + void *p = malloc(size);

414 + if (unlikely(!p)) {

415 + OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

416 + fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", size, sm.ram_available_bytes);

417 + }

418 + return p;

419 +}

420 +

421 +void *callocz(size_t nmemb, size_t size) {

422 + void *p = calloc(nmemb, size);

423 + if (unlikely(!p)) {

424 + OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

425 + fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", nmemb * size, sm.ram_available_bytes);

426 + }

427 + return p;

428 +}

429 +

430 +void *reallocz(void *ptr, size_t size) {

431 + void *p = realloc(ptr, size);

432 + if (unlikely(!p)) {

433 + OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

434 + fatal("Cannot re-allocate memory to %zu bytes. (system memory available bytes: %lu)", size, sm.ram_available_bytes);

435 + }

436 + return p;

437 +}

438 +

439 +void posix_memfree(void *ptr) {

440 + free(ptr);

441 +}

442 +#endif

443 +

444 +void mallocz_release_as_much_memory_to_the_system(void) {

445 +#if defined(HAVE_C_MALLOC_TRIM)

446 + static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

447 + if(spinlock_trylock(&spinlock)) {

448 + malloc_trim(0);

449 + spinlock_unlock(&spinlock);

450 + }

451 +#endif

452 +}

src/libnetdata/memory/nd-mallocz.h new

+65

@@ -0,0 +1,65 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_ND_MALLOCZ_H

4 +#define NETDATA_ND_MALLOCZ_H

5 +

6 +#include "../common.h"

7 +

8 +// memory allocation functions that handle failures

9 +#ifdef NETDATA_TRACE_ALLOCATIONS

10 +struct malloc_trace {

11 + avl_t avl;

12 +

13 + const char *function;

14 + const char *file;

15 + size_t line;

16 +

17 + size_t malloc_calls;

18 + size_t calloc_calls;

19 + size_t realloc_calls;

20 + size_t strdup_calls;

21 + size_t free_calls;

22 +

23 + size_t mmap_calls;

24 + size_t munmap_calls;

25 +

26 + size_t allocations;

27 + size_t bytes;

28 +

29 + struct rrddim *rd_bytes;

30 + struct rrddim *rd_allocations;

31 + struct rrddim *rd_avg_alloc;

32 + struct rrddim *rd_ops;

33 +};

34 +

35 +int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data);

36 +

37 +#define strdupz(s) strdupz_int(s, __FILE__, __FUNCTION__, __LINE__)

38 +#define strndupz(s, len) strndupz_int(s, len, __FILE__, __FUNCTION__, __LINE__)

39 +#define callocz(nmemb, size) callocz_int(nmemb, size, __FILE__, __FUNCTION__, __LINE__)

40 +#define mallocz(size) mallocz_int(size, __FILE__, __FUNCTION__, __LINE__)

41 +#define reallocz(ptr, size) reallocz_int(ptr, size, __FILE__, __FUNCTION__, __LINE__)

42 +#define freez(ptr) freez_int(ptr, __FILE__, __FUNCTION__, __LINE__)

43 +#define mallocz_usable_size(ptr) mallocz_usable_size_int(ptr, __FILE__, __FUNCTION__, __LINE__)

44 +

45 +char *strdupz_int(const char *s, const char *file, const char *function, size_t line);

46 +char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line);

47 +void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line);

48 +void *mallocz_int(size_t size, const char *file, const char *function, size_t line);

49 +void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line);

50 +void freez_int(void *ptr, const char *file, const char *function, size_t line);

51 +size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line);

52 +

53 +#else // NETDATA_TRACE_ALLOCATIONS

54 +char *strdupz(const char *s) MALLOCLIKE NEVERNULL;

55 +char *strndupz(const char *s, size_t len) MALLOCLIKE NEVERNULL;

56 +void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;

57 +void *mallocz(size_t size) MALLOCLIKE NEVERNULL;

58 +void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL;

59 +void freez(void *ptr);

60 +#endif // NETDATA_TRACE_ALLOCATIONS

61 +

62 +void mallocz_release_as_much_memory_to_the_system(void);

63 +void posix_memfree(void *ptr);

64 +

65 +#endif //NETDATA_ND_MALLOCZ_H

src/libnetdata/memory/nd-mmap.c new

+216

@@ -0,0 +1,216 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "../libnetdata.h"

4 +

5 +size_t nd_mmap_count = 0;

6 +size_t nd_mmap_size = 0;

7 +

8 +#if !defined(MADV_DONTFORK)

9 +#define MADV_DONTFORK 0

10 +#endif

11 +

12 +#if !defined(O_NOATIME)

13 +#define O_NOATIME 0

14 +#endif

15 +

16 +#if defined(MADV_MERGEABLE)

17 +int enable_ksm = CONFIG_BOOLEAN_AUTO;

18 +#else

19 +int enable_ksm = 0;

20 +#endif

21 +

22 +static int memory_file_open(const char *filename, size_t size) {

23 + // netdata_log_info("memory_file_open('%s', %zu", filename, size);

24 +

25 + int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME | O_CLOEXEC, 0664);

26 + if (fd != -1) {

27 + if (lseek(fd, size, SEEK_SET) == (off_t) size) {

28 + if (write(fd, "", 1) == 1) {

29 + if (ftruncate(fd, size))

30 + netdata_log_error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size);

31 + }

32 + else

33 + netdata_log_error("Cannot write to file '%s' at position %zu.", filename, size);

34 + }

35 + else

36 + netdata_log_error("Cannot seek file '%s' to size %zu.", filename, size);

37 + }

38 + else

39 + netdata_log_error("Cannot create/open file '%s'.", filename);

40 +

41 + return fd;

42 +}

43 +

44 +inline int madvise_sequential(void *mem, size_t len) {

45 + static int logger = 1;

46 + int ret = madvise(mem, len, MADV_SEQUENTIAL);

47 +

48 + if (ret != 0 && logger-- > 0)

49 + netdata_log_error("madvise(MADV_SEQUENTIAL) of size %zu, failed.", len);

50 + return ret;

51 +}

52 +

53 +inline int madvise_random(void *mem, size_t len) {

54 + static int logger = 1;

55 + int ret = madvise(mem, len, MADV_RANDOM);

56 +

57 + if (ret != 0 && logger-- > 0)

58 + netdata_log_error("madvise(MADV_RANDOM) of size %zu, failed.", len);

59 + return ret;

60 +}

61 +

62 +inline int madvise_dontfork(void *mem, size_t len) {

63 + static int logger = 1;

64 + int ret = madvise(mem, len, MADV_DONTFORK);

65 +

66 + if (ret != 0 && logger-- > 0)

67 + netdata_log_error("madvise(MADV_DONTFORK) of size %zu, failed.", len);

68 + return ret;

69 +}

70 +

71 +inline int madvise_willneed(void *mem, size_t len) {

72 + static int logger = 1;

73 + int ret = madvise(mem, len, MADV_WILLNEED);

74 +

75 + if (ret != 0 && logger-- > 0)

76 + netdata_log_error("madvise(MADV_WILLNEED) of size %zu, failed.", len);

77 + return ret;

78 +}

79 +

80 +inline int madvise_dontneed(void *mem, size_t len) {

81 + static int logger = 1;

82 + int ret = madvise(mem, len, MADV_DONTNEED);

83 +

84 + if (ret != 0 && logger-- > 0)

85 + netdata_log_error("madvise(MADV_DONTNEED) of size %zu, failed.", len);

86 + return ret;

87 +}

88 +

89 +inline int madvise_dontdump(void *mem __maybe_unused, size_t len __maybe_unused) {

90 +#if __linux__

91 + static int logger = 1;

92 + int ret = madvise(mem, len, MADV_DONTDUMP);

93 +

94 + if (ret != 0 && logger-- > 0)

95 + netdata_log_error("madvise(MADV_DONTDUMP) of size %zu, failed.", len);

96 + return ret;

97 +#else

98 + return 0;

99 +#endif

100 +}

101 +

102 +inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused) {

103 +#ifdef MADV_MERGEABLE

104 + static int logger = 1;

105 + int ret = madvise(mem, len, MADV_MERGEABLE);

106 +

107 + if (ret != 0 && logger-- > 0)

108 + netdata_log_error("madvise(MADV_MERGEABLE) of size %zu, failed.", len);

109 + return ret;

110 +#else

111 + return 0;

112 +#endif

113 +}

114 +

115 +int nd_munmap(void *ptr, size_t size) {

116 +#ifdef NETDATA_TRACE_ALLOCATIONS

117 + malloc_trace_munmap(size);

118 +#endif

119 +

120 + int rc = munmap(ptr, size);

121 +

122 + if(rc == 0) {

123 + __atomic_sub_fetch(&nd_mmap_count, 1, __ATOMIC_RELAXED);

124 + __atomic_sub_fetch(&nd_mmap_size, size, __ATOMIC_RELAXED);

125 + }

126 +

127 + return rc;

128 +}

129 +

130 +void *nd_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) {

131 + void *rc = mmap(addr, len, prot, flags, fd, offset);

132 +

133 + if(rc != MAP_FAILED) {

134 + __atomic_add_fetch(&nd_mmap_count, 1, __ATOMIC_RELAXED);

135 + __atomic_add_fetch(&nd_mmap_size, len, __ATOMIC_RELAXED);

136 +

137 +#ifdef NETDATA_TRACE_ALLOCATIONS

138 + malloc_trace_mmap(size);

139 +#endif

140 + }

141 +

142 + return rc;

143 +}

144 +

145 +void *nd_mmap_advanced(const char *filename, size_t size, int flags, int ksm, bool read_only, bool dont_dump, int *open_fd) {

146 + // netdata_log_info("netdata_mmap('%s', %zu", filename, size);

147 +

148 + // MAP_SHARED is used in memory mode map

149 + // MAP_PRIVATE is used in memory mode ram and save

150 +

151 + if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE)))

152 + fatal("Neither MAP_SHARED or MAP_PRIVATE were given to nd_mmap_advanced()");

153 +

154 + if(unlikely((flags & MAP_SHARED) && (flags & MAP_PRIVATE)))

155 + fatal("Both MAP_SHARED and MAP_PRIVATE were given to nd_mmap_advanced()");

156 +

157 + if(unlikely((flags & MAP_SHARED) && (!filename || !*filename)))

158 + fatal("MAP_SHARED requested, without a filename to nd_mmap_advanced()");

159 +

160 + // don't enable ksm is the global setting is disabled

161 + if(unlikely(!enable_ksm)) ksm = 0;

162 +

163 + // KSM only merges anonymous (private) pages, never pagecache (file) pages

164 + // but MAP_PRIVATE without MAP_ANONYMOUS it fails too, so we need it always

165 + if((flags & MAP_PRIVATE)) flags |= MAP_ANONYMOUS;

166 +

167 + int fd = -1;

168 + void *mem = MAP_FAILED;

169 +

170 + if(filename && *filename) {

171 + // open/create the file to be used

172 + fd = memory_file_open(filename, size);

173 + if(fd == -1) goto cleanup;

174 + }

175 +

176 + int fd_for_mmap = fd;

177 + if(fd != -1 && (flags & MAP_PRIVATE)) {

178 + // this is MAP_PRIVATE allocation

179 + // no need for mmap() to use our fd

180 + // we will copy the file into the memory allocated

181 + fd_for_mmap = -1;

182 + }

183 +

184 + mem = nd_mmap(NULL, size, read_only ? PROT_READ : PROT_READ | PROT_WRITE, flags, fd_for_mmap, 0);

185 + if (mem != MAP_FAILED) {

186 + // if we have a file open, but we didn't give it to mmap(),

187 + // we have to read the file into the memory block we allocated

188 + if(fd != -1 && fd_for_mmap == -1) {

189 + if (lseek(fd, 0, SEEK_SET) == 0) {

190 + if (read(fd, mem, size) != (ssize_t) size)

191 + netdata_log_info("Cannot read from file '%s'", filename);

192 + }

193 + else netdata_log_info("Cannot seek to beginning of file '%s'.", filename);

194 + }

195 +

196 + // madvise_sequential(mem, size);

197 + // madvise_dontfork(mem, size); // aral is initialized before we daemonize

198 + if(dont_dump) madvise_dontdump(mem, size);

199 + // if(flags & MAP_SHARED) madvise_willneed(mem, size);

200 + if(ksm) madvise_mergeable(mem, size);

201 + }

202 +

203 +cleanup:

204 + if(fd != -1) {

205 + if (open_fd)

206 + *open_fd = fd;

207 + else

208 + close(fd);

209 + }

210 +

211 + if(mem == MAP_FAILED)

212 + return NULL;

213 +

214 + errno_clear();

215 + return mem;

216 +}

src/libnetdata/memory/nd-mmap.h new

+24

@@ -0,0 +1,24 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_ND_MMAP_H

4 +#define NETDATA_ND_MMAP_H

5 +

6 +#include "../common.h"

7 +

8 +int madvise_sequential(void *mem, size_t len);

9 +int madvise_random(void *mem, size_t len);

10 +int madvise_dontfork(void *mem, size_t len);

11 +int madvise_willneed(void *mem, size_t len);

12 +int madvise_dontneed(void *mem, size_t len);

13 +int madvise_dontdump(void *mem, size_t len);

14 +int madvise_mergeable(void *mem, size_t len);

15 +

16 +extern size_t nd_mmap_count;

17 +extern size_t nd_mmap_size;

18 +extern int enable_ksm;

19 +

20 +void *nd_mmap_advanced(const char *filename, size_t size, int flags, int ksm, bool read_only, bool dont_dump, int *open_fd);

21 +void *nd_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset);

22 +int nd_munmap(void *ptr, size_t size);

23 +

24 +#endif //NETDATA_ND_MMAP_H

src/libnetdata/onewayalloc/onewayalloc.c

+3 -23

@@ -1,8 +1,5 @@

1 #include "onewayalloc.h"

3 -// https://www.gnu.org/software/libc/manual/html_node/Aligned-Memory-Blocks.html

4 -#define OWA_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)

5 -

3 typedef struct owa_page {

4 size_t stats_pages;

5 size_t stats_pages_size;

@@ -20,29 +17,12 @@ size_t onewayalloc_allocated_memory(void) {

17 return __atomic_load_n(&onewayalloc_total_memory, __ATOMIC_RELAXED);

18 }

23 -// allocations need to be aligned to CPU register width

24 -// https://en.wikipedia.org/wiki/Data_structure_alignment

25 -static inline size_t natural_alignment(size_t size) {

26 - if(unlikely(size % OWA_NATURAL_ALIGNMENT))

27 - size = size + OWA_NATURAL_ALIGNMENT - (size % OWA_NATURAL_ALIGNMENT);

28 -

29 - return size;

30 -}

31 -

20 // Create an OWA

21 // Once it is created, the caller may call the onewayalloc_mallocz()

22 // any number of times, for any amount of memory.

24 static OWA_PAGE *onewayalloc_create_internal(OWA_PAGE *head, size_t size_hint) {

37 - static size_t OWA_NATURAL_PAGE_SIZE = 0;

38 -

39 - if(unlikely(!OWA_NATURAL_PAGE_SIZE)) {

40 - long int page_size = sysconf(_SC_PAGE_SIZE);

41 - if (unlikely(page_size == -1))

42 - OWA_NATURAL_PAGE_SIZE = 4096;

43 - else

44 - OWA_NATURAL_PAGE_SIZE = page_size;

45 - }

25 + size_t OWA_NATURAL_PAGE_SIZE = os_get_system_page_size();

27 // our default page size

28 size_t size = 32768;

@@ -73,7 +53,7 @@ static OWA_PAGE *onewayalloc_create_internal(OWA_PAGE *head, size_t size_hint) {

53 size = size + OWA_NATURAL_PAGE_SIZE - (size % OWA_NATURAL_PAGE_SIZE);

55 // Use netdata_mmap instead of mallocz

76 - OWA_PAGE *page = (OWA_PAGE *)netdata_mmap(NULL, size, MAP_ANONYMOUS|MAP_PRIVATE, 0, false, false, NULL);

56 + OWA_PAGE *page = (OWA_PAGE *)nd_mmap_advanced(NULL, size, MAP_ANONYMOUS | MAP_PRIVATE, 0, false, false, NULL);

57 if(unlikely(!page)) fatal("Cannot allocate onewayalloc buffer of size %zu", size);

59 __atomic_add_fetch(&onewayalloc_total_memory, size, __ATOMIC_RELAXED);

@@ -216,7 +196,7 @@ void onewayalloc_destroy(ONEWAYALLOC *owa) {

196 page = page->next;

197

198 // Use netdata_munmap instead of freez

219 - netdata_munmap(p, p->size);

199 + nd_munmap(p, p->size);

200 }

201

202 __atomic_sub_fetch(&onewayalloc_total_memory, total_size, __ATOMIC_RELAXED);

src/libnetdata/os/get_system_pagesize.c new

+36

@@ -0,0 +1,36 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "get_system_pagesize.h"

4 +

5 +#if defined(OS_WINDOWS)

6 +#include <windows.h>

7 +

8 +size_t os_get_system_page_size(void) {

9 + static size_t page_size = 0;

10 +

11 + if (!page_size) {

12 + SYSTEM_INFO sys_info;

13 + GetSystemInfo(&sys_info);

14 +

15 + // Set the page size, default to 4096 if for some reason it's invalid

16 + page_size = (sys_info.dwPageSize >= 4096) ? sys_info.dwPageSize : 4096;

17 + }

18 +

19 + return page_size;

20 +}

21 +

22 +#else

23 +

24 +size_t os_get_system_page_size(void) {

25 + static long int page_size = 0;

26 +

27 + if(unlikely(!page_size)) {

28 + page_size = sysconf(_SC_PAGE_SIZE);

29 + if (unlikely(page_size <= 4096))

30 + page_size = 4096;

31 + }

32 +

33 + return page_size;

34 +}

35 +

36 +#endif

\ No newline at end of file

src/libnetdata/os/get_system_pagesize.h new

+10

@@ -0,0 +1,10 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_GET_SYSTEM_PAGESIZE_H

4 +#define NETDATA_GET_SYSTEM_PAGESIZE_H

5 +

6 +#include "../common.h"

7 +

8 +size_t os_get_system_page_size(void) __attribute__((const));

9 +

10 +#endif //NETDATA_GET_SYSTEM_PAGESIZE_H

src/libnetdata/os/os.h

@@ -19,6 +19,7 @@

19 #include "gettid.h"

20 #include "get_pid_max.h"

21 #include "get_system_cpus.h"

22 +#include "get_system_pagesize.h"

23 #include "sleep.h"

24 #include "uuid_generate.h"

25 #include "setenv.h"