@cryptotaxi247 / netdata-1
Code Commits Issues Pulls Sessions CI/CD
master
master
c 510 lines 15.9 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "../libnetdata.h"
4
5 out_of_memory_cb out_of_memory_callback = NULL;
6 void mallocz_register_out_of_memory_cb(out_of_memory_cb cb) {
7 out_of_memory_callback = cb;
8 }
9
10
11 ALWAYS_INLINE NORETURN
12 void out_of_memory(const char *call, size_t size, const char *details) {
13 int errno_saved = errno;
14 exit_initiated_add(EXIT_REASON_OUT_OF_MEMORY);
15
16 if(out_of_memory_callback)
17 out_of_memory_callback();
18
19 OS_PROCESS_MEMORY proc_mem = os_process_memory(0);
20 uint64_t max_rss = OS_PROCESS_MEMORY_OK(proc_mem) ? proc_mem.max_rss : 0;
21
22 char mem_available[64];
23 char rss_used[64];
24
25 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
26 size_snprintf(mem_available, sizeof(mem_available), sm.ram_available_bytes, "B", false);
27 size_snprintf(rss_used, sizeof(rss_used), max_rss, "B", false);
28
29 errno = errno_saved;
30 fatal("Out of memory on %s(%zu bytes)!\n"
31 "System memory available: %s, while our max RSS usage is: %s\n"
32 "O/S mmap limit: %llu, while our mmap count is: %zu\n"
33 "Additional details: %s",
34 call, size,
35 mem_available, rss_used,
36 os_mmap_limit(), __atomic_load_n(&nd_mmap_count, __ATOMIC_RELAXED),
37 details ? details : "none");
38 }
39
40 // ----------------------------------------------------------------------------
41 // memory allocation functions that handle failures
42
43 // although netdata does not use memory allocations too often (netdata tries to
44 // maintain its memory footprint stable during runtime, i.e. all buffers are
45 // allocated during initialization and are adapted to current use throughout
46 // its lifetime), these can be used to override the default system allocation
47 // routines.
48
49 #ifdef NETDATA_TRACE_ALLOCATIONS
50 #warning NETDATA_TRACE_ALLOCATIONS ENABLED
51 #include "Judy.h"
52
53 #if defined(HAVE_DLSYM) && defined(ENABLE_DLSYM)
54 #include <dlfcn.h>
55
56 typedef void (*libc_function_t)(void);
57
58 static void *malloc_first_run(size_t size);
59 static void *(*libc_malloc)(size_t) = malloc_first_run;
60
61 static void *calloc_first_run(size_t n, size_t size);
62 static void *(*libc_calloc)(size_t, size_t) = calloc_first_run;
63
64 static void *realloc_first_run(void *ptr, size_t size);
65 static void *(*libc_realloc)(void *, size_t) = realloc_first_run;
66
67 static void free_first_run(void *ptr);
68 static void (*libc_free)(void *) = free_first_run;
69
70 static char *strdup_first_run(const char *s);
71 static char *(*libc_strdup)(const char *) = strdup_first_run;
72
73 static char *strndup_first_run(const char *s, size_t len);
74 static char *(*libc_strndup)(const char *, size_t) = strndup_first_run;
75
76 static size_t malloc_usable_size_first_run(void *ptr);
77 #ifdef HAVE_MALLOC_USABLE_SIZE
78 static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size_first_run;
79 #else
80 static size_t (*libc_malloc_usable_size)(void *) = NULL;
81 #endif
82
83 static void link_system_library_function(libc_function_t *func_pptr, const char *name, bool required) {
84 *func_pptr = dlsym(RTLD_NEXT, name);
85 if(!*func_pptr && required) {
86 fprintf(stderr, "FATAL: Cannot find system's %s() function.\n", name);
87 abort();
88 }
89 }
90
91 static void *malloc_first_run(size_t size) {
92 link_system_library_function((libc_function_t *) &libc_malloc, "malloc", true);
93 return libc_malloc(size);
94 }
95
96 static void *calloc_first_run(size_t n, size_t size) {
97 link_system_library_function((libc_function_t *) &libc_calloc, "calloc", true);
98 return libc_calloc(n, size);
99 }
100
101 static void *realloc_first_run(void *ptr, size_t size) {
102 link_system_library_function((libc_function_t *) &libc_realloc, "realloc", true);
103 return libc_realloc(ptr, size);
104 }
105
106 static void free_first_run(void *ptr) {
107 link_system_library_function((libc_function_t *) &libc_free, "free", true);
108 libc_free(ptr);
109 }
110
111 static char *strdup_first_run(const char *s) {
112 link_system_library_function((libc_function_t *) &libc_strdup, "strdup", true);
113 return libc_strdup(s);
114 }
115
116 static char *strndup_first_run(const char *s, size_t len) {
117 link_system_library_function((libc_function_t *) &libc_strndup, "strndup", true);
118 return libc_strndup(s, len);
119 }
120
121 static size_t malloc_usable_size_first_run(void *ptr) {
122 link_system_library_function((libc_function_t *) &libc_malloc_usable_size, "malloc_usable_size", false);
123
124 if(libc_malloc_usable_size)
125 return libc_malloc_usable_size(ptr);
126 else
127 return 0;
128 }
129
130 void *malloc(size_t size) {
131 return mallocz(size);
132 }
133
134 void *calloc(size_t n, size_t size) {
135 return callocz(n, size);
136 }
137
138 void *realloc(void *ptr, size_t size) {
139 return reallocz(ptr, size);
140 }
141
142 void *reallocarray(void *ptr, size_t n, size_t size) {
143 return reallocz(ptr, n * size);
144 }
145
146 void free(void *ptr) {
147 freez(ptr);
148 }
149
150 char *strdup(const char *s) {
151 return strdupz(s);
152 }
153
154 char *strndup(const char *s, size_t len) {
155 return strndupz(s, len);
156 }
157
158 size_t malloc_usable_size(void *ptr) {
159 return mallocz_usable_size(ptr);
160 }
161 #else // !HAVE_DLSYM
162
163 static void *(*libc_malloc)(size_t) = malloc;
164 static void *(*libc_calloc)(size_t, size_t) = calloc;
165 static void *(*libc_realloc)(void *, size_t) = realloc;
166 static void (*libc_free)(void *) = free;
167
168 #ifdef HAVE_MALLOC_USABLE_SIZE
169 static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size;
170 #else
171 static size_t (*libc_malloc_usable_size)(void *) = NULL;
172 #endif
173
174 #endif // HAVE_DLSYM
175
176
177 void posix_memfree(void *ptr) {
178 libc_free(ptr);
179 }
180
181 struct malloc_header_signature {
182 uint32_t magic;
183 uint32_t size;
184 struct malloc_trace *trace;
185 };
186
187 struct malloc_header {
188 struct malloc_header_signature signature;
189 uint8_t padding[(sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) ? MALLOC_ALIGNMENT - (sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) : 0];
190 uint8_t data[];
191 };
192
193 static size_t malloc_header_size = sizeof(struct malloc_header);
194
195 int malloc_trace_compare(void *A, void *B) {
196 struct malloc_trace *a = A;
197 struct malloc_trace *b = B;
198 return strcmp(a->function, b->function);
199 }
200
201 static avl_tree_lock malloc_trace_index = {
202 .avl_tree = {
203 .root = NULL,
204 .compar = malloc_trace_compare},
205 .rwlock = AVL_LOCK_INITIALIZER
206 };
207
208 int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data) {
209 return avl_traverse_lock(&malloc_trace_index, callback, data);
210 }
211
212 NEVERNULL WARNUNUSED
213 static struct malloc_trace *malloc_trace_find_or_create(const char *file, const char *function, size_t line) {
214 struct malloc_trace tmp = {
215 .line = line,
216 .function = function,
217 .file = file,
218 };
219
220 struct malloc_trace *t = (struct malloc_trace *)avl_search_lock(&malloc_trace_index, (avl_t *)&tmp);
221 if(!t) {
222 t = libc_calloc(1, sizeof(struct malloc_trace));
223 if(!t) fatal("No memory");
224 t->line = line;
225 t->function = function;
226 t->file = file;
227
228 struct malloc_trace *t2 = (struct malloc_trace *)avl_insert_lock(&malloc_trace_index, (avl_t *)t);
229 if(t2 != t)
230 free(t);
231
232 t = t2;
233 }
234
235 if(!t)
236 fatal("Cannot insert to AVL");
237
238 return t;
239 }
240
241 void malloc_trace_mmap(size_t size) {
242 struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);
243 size_t_atomic_count(add, p->mmap_calls, 1);
244 size_t_atomic_count(add, p->allocations, 1);
245 size_t_atomic_bytes(add, p->bytes, size);
246 }
247
248 void malloc_trace_munmap(size_t size) {
249 struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);
250 size_t_atomic_count(add, p->munmap_calls, 1);
251 size_t_atomic_count(sub, p->allocations, 1);
252 size_t_atomic_bytes(sub, p->bytes, size);
253 }
254
255 void *mallocz_int(size_t size, const char *file, const char *function, size_t line) {
256 struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
257
258 size_t_atomic_count(add, p->malloc_calls, 1);
259 size_t_atomic_count(add, p->allocations, 1);
260 size_t_atomic_bytes(add, p->bytes, size);
261
262 struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);
263 if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
264 t->signature.magic = 0x0BADCAFE;
265 t->signature.trace = p;
266 t->signature.size = size;
267
268 #ifdef NETDATA_INTERNAL_CHECKS
269 for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
270 t->padding[i] = 0xFF;
271 #endif
272
273 return (void *)&t->data;
274 }
275
276 void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line) {
277 struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
278 size = nmemb * size;
279
280 size_t_atomic_count(add, p->calloc_calls, 1);
281 size_t_atomic_count(add, p->allocations, 1);
282 size_t_atomic_bytes(add, p->bytes, size);
283
284 struct malloc_header *t = (struct malloc_header *)libc_calloc(1, malloc_header_size + size);
285 if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
286 t->signature.magic = 0x0BADCAFE;
287 t->signature.trace = p;
288 t->signature.size = size;
289
290 #ifdef NETDATA_INTERNAL_CHECKS
291 for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
292 t->padding[i] = 0xFF;
293 #endif
294
295 return &t->data;
296 }
297
298 char *strdupz_int(const char *s, const char *file, const char *function, size_t line) {
299 struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
300 size_t size = strlen(s) + 1;
301
302 size_t_atomic_count(add, p->strdup_calls, 1);
303 size_t_atomic_count(add, p->allocations, 1);
304 size_t_atomic_bytes(add, p->bytes, size);
305
306 struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);
307 if (unlikely(!t)) fatal("strdupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
308 t->signature.magic = 0x0BADCAFE;
309 t->signature.trace = p;
310 t->signature.size = size;
311
312 #ifdef NETDATA_INTERNAL_CHECKS
313 for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
314 t->padding[i] = 0xFF;
315 #endif
316
317 memcpy(&t->data, s, size);
318 return (char *)&t->data;
319 }
320
321 char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line) {
322 struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
323 size_t size = len + 1;
324
325 size_t_atomic_count(add, p->strdup_calls, 1);
326 size_t_atomic_count(add, p->allocations, 1);
327 size_t_atomic_bytes(add, p->bytes, size);
328
329 struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);
330 if (unlikely(!t)) fatal("strndupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
331 t->signature.magic = 0x0BADCAFE;
332 t->signature.trace = p;
333 t->signature.size = size;
334
335 #ifdef NETDATA_INTERNAL_CHECKS
336 for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
337 t->padding[i] = 0xFF;
338 #endif
339
340 memcpy(&t->data, s, size);
341 t->data[len] = '\0';
342 return (char *)&t->data;
343 }
344
345 static struct malloc_header *malloc_get_header(void *ptr, const char *caller, const char *file, const char *function, size_t line) {
346 uint8_t *ret = (uint8_t *)ptr - malloc_header_size;
347 struct malloc_header *t = (struct malloc_header *)ret;
348
349 if(t->signature.magic != 0x0BADCAFE) {
350 netdata_log_error("pointer %p is not our pointer (called %s() from %zu@%s, %s()).", ptr, caller, line, file, function);
351 return NULL;
352 }
353
354 return t;
355 }
356
357 void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line) {
358 if(!ptr) return mallocz_int(size, file, function, line);
359
360 struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);
361 if(!t)
362 return libc_realloc(ptr, size);
363
364 if(t->signature.size == size) return ptr;
365 size_t_atomic_count(add, t->signature.trace->free_calls, 1);
366 size_t_atomic_count(sub, t->signature.trace->allocations, 1);
367 size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size);
368
369 struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
370 size_t_atomic_count(add, p->realloc_calls, 1);
371 size_t_atomic_count(add, p->allocations, 1);
372 size_t_atomic_bytes(add, p->bytes, size);
373
374 t = (struct malloc_header *)libc_realloc(t, malloc_header_size + size);
375 if (unlikely(!t)) fatal("reallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
376 t->signature.magic = 0x0BADCAFE;
377 t->signature.trace = p;
378 t->signature.size = size;
379
380 #ifdef NETDATA_INTERNAL_CHECKS
381 for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
382 t->padding[i] = 0xFF;
383 #endif
384
385 return (void *)&t->data;
386 }
387
388 size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line) {
389 if(unlikely(!ptr)) return 0;
390
391 struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);
392 if(!t) {
393 if(libc_malloc_usable_size)
394 return libc_malloc_usable_size(ptr);
395 else
396 return 0;
397 }
398
399 return t->signature.size;
400 }
401
402 void freez_int(void *ptr, const char *file, const char *function, size_t line) {
403 if(unlikely(!ptr)) return;
404
405 struct malloc_header *t = malloc_get_header(ptr, __FUNCTION__, file, function, line);
406 if(!t) {
407 libc_free(ptr);
408 return;
409 }
410
411 size_t_atomic_count(add, t->signature.trace->free_calls, 1);
412 size_t_atomic_count(sub, t->signature.trace->allocations, 1);
413 size_t_atomic_bytes(sub, t->signature.trace->bytes, t->signature.size);
414
415 #ifdef NETDATA_INTERNAL_CHECKS
416 // it should crash if it is used after freeing it
417 memset(t, 0, malloc_header_size + t->signature.size);
418 #endif
419
420 libc_free(t);
421 }
422 #else
423
424 ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED
425 char *strdupz(const char *s) {
426 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_STRDUP);
427
428 char *t = strdup(s);
429 if (unlikely(!t))
430 out_of_memory(__FUNCTION__ , strlen(s) + 1, NULL);
431
432 return t;
433 }
434
435 ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED
436 char *strndupz(const char *s, size_t len) {
437 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_STRNDUP);
438
439 char *t = strndup(s, len);
440 if (unlikely(!t))
441 out_of_memory(__FUNCTION__ , len + 1, NULL);
442
443 return t;
444 }
445
446 // If ptr is NULL, no operation is performed.
447 ALWAYS_INLINE
448 void freez(void *ptr) {
449 if(likely(ptr)) {
450 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_FREE);
451 free(ptr);
452 }
453 }
454
455 ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED
456 void *mallocz(size_t size) {
457 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_MALLOC);
458 void *p = malloc(size);
459 if (unlikely(!p))
460 out_of_memory(__FUNCTION__, size, NULL);
461
462 return p;
463 }
464
465 ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED
466 void *callocz(size_t nmemb, size_t size) {
467 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_CALLOC);
468 void *p = calloc(nmemb, size);
469 if (unlikely(!p))
470 out_of_memory(__FUNCTION__, nmemb * size, NULL);
471
472 return p;
473 }
474
475 ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED
476 void *reallocz(void *ptr, size_t size) {
477 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_REALLOC);
478 void *p = realloc(ptr, size);
479 if (unlikely(!p))
480 out_of_memory(__FUNCTION__, size, NULL);
481
482 return p;
483 }
484
485 ALWAYS_INLINE
486 int posix_memalignz(void **memptr, size_t alignment, size_t size) {
487 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN);
488 int rc = posix_memalign(memptr, alignment, size);
489 if(unlikely(rc))
490 out_of_memory(__FUNCTION__, size, NULL);
491
492 return rc;
493 }
494
495 ALWAYS_INLINE
496 void posix_memalign_freez(void *ptr) {
497 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN_FREE);
498 free(ptr);
499 }
500 #endif
501
502 void mallocz_release_as_much_memory_to_the_system(void) {
503 #if defined(HAVE_C_MALLOC_TRIM)
504 static SPINLOCK spinlock = SPINLOCK_INITIALIZER;
505 if(spinlock_trylock(&spinlock)) {
506 malloc_trim(0);
507 spinlock_unlock(&spinlock);
508 }
509 #endif
510 }