Abort on fatal and report system available bytes on allocation failures. (#19332)
* Abort on non-zero rc when we have sentry. * Abort on recursive-fatals when sentry is enabled * Add function that returns last reported system memory. * Provide available memory in fatal message. * Update system_memory.c --------- Co-authored-by: Costa Tsaousis <costa@netdata.cloud>
vkalintiris committed
Jan 8, 2025 at 11:15 UTC
888c37ca9b5004a7c5b5860008f01f242823278e
5 files changed
+51
-9
src/daemon/daemon-shutdown.c
+8
-3
@@ -296,8 +296,13 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
296
#endif
297
298
#ifdef ENABLE_SENTRY
299
- nd_sentry_fini();
300
-#endif
301
-
299
+ if (ret) {
300
+ abort();
301
+ } else {
302
+ nd_sentry_fini();
303
+ exit(ret);
304
+ }
305
+#else
306
exit(ret);
307
+#endif
308
}
src/libnetdata/libnetdata.c
+20
-5
@@ -410,13 +410,19 @@ void freez_int(void *ptr, const char *file, const char *function, size_t line) {
410
411
char *strdupz(const char *s) {
412
char *t = strdup(s);
413
- if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s);
413
+ if (unlikely(!t)) {
414
+ OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
415
+ fatal("Cannot strdup() string '%s' (system memory available bytes: %lu)", s, sm.ram_available_bytes);
416
+ }
417
return t;
418
}
419
420
char *strndupz(const char *s, size_t len) {
421
char *t = strndup(s, len);
419
- if (unlikely(!t)) fatal("Cannot strndup() string '%s' of len %zu", s, len);
422
+ if (unlikely(!t)) {
423
+ OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
424
+ fatal("Cannot strndup() string '%s' of len %zu (system memory available bytes: %lu)", s, len, sm.ram_available_bytes);
425
+ }
426
return t;
427
}
428
@@ -427,19 +433,28 @@ void freez(void *ptr) {
433
434
void *mallocz(size_t size) {
435
void *p = malloc(size);
430
- if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size);
436
+ if (unlikely(!p)) {
437
+ OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
438
+ fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", size, sm.ram_available_bytes);
439
+ }
440
return p;
441
}
442
443
void *callocz(size_t nmemb, size_t size) {
444
void *p = calloc(nmemb, size);
436
- if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size);
445
+ if (unlikely(!p)) {
446
+ OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
447
+ fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", nmemb * size, sm.ram_available_bytes);
448
+ }
449
return p;
450
}
451
452
void *reallocz(void *ptr, size_t size) {
453
void *p = realloc(ptr, size);
442
- if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size);
454
+ if (unlikely(!p)) {
455
+ OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
456
+ fatal("Cannot re-allocate memory to %zu bytes. (system memory available bytes: %lu)", size, sm.ram_available_bytes);
457
+ }
458
return p;
459
}
460
src/libnetdata/log/nd_log.c
+5
@@ -419,7 +419,12 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned
419
fprintf(stderr, "\nRECURSIVE FATAL STATEMENTS, latest from %s() of %lu@%s, EXITING NOW! 23e93dfccbf64e11aac858b9410d8a82\n",
420
function, line, file);
421
fflush(stderr);
422
+
423
+#ifdef ENABLE_SENTRY
424
+ abort();
425
+#else
426
_exit(1);
427
+#endif
428
}
429
430
int saved_errno = errno;
src/libnetdata/os/system_memory.c
+15
-1
@@ -2,6 +2,14 @@
2
3
#include "libnetdata/libnetdata.h"
4
5
+static OS_SYSTEM_MEMORY os_system_memory_last = {
6
+ 0, 0,
7
+};
8
+
9
+OS_SYSTEM_MEMORY os_last_reported_system_memory(void) {
10
+ return os_system_memory_last;
11
+}
12
+
13
// Windows
14
#if defined(OS_WINDOWS)
15
#include <windows.h>
@@ -14,6 +22,7 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram __maybe_unused) {
22
if (GlobalMemoryStatusEx(&statex)) {
23
sm.ram_total_bytes = statex.ullTotalPhys;
24
sm.ram_available_bytes = statex.ullAvailPhys;
25
+ os_system_memory_last = sm;
26
}
27
28
return sm;
@@ -56,10 +65,11 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {
65
mach_port_deallocate(mach_task_self(), mach_port);
66
}
67
59
- return (OS_SYSTEM_MEMORY){
68
+ os_system_memory_last = (OS_SYSTEM_MEMORY){
69
.ram_total_bytes = total_ram,
70
.ram_available_bytes = ram_available,
71
};
72
+ return os_system_memory_last;
73
}
74
#endif
75
@@ -119,6 +129,7 @@ static OS_SYSTEM_MEMORY os_system_memory_cgroup_v1(bool query_total_ram __maybe_
129
130
done:
131
sm.ram_available_bytes = sm.ram_total_bytes - (used - inactive);
132
+ os_system_memory_last = sm;
133
return sm;
134
135
failed:
@@ -184,6 +195,7 @@ static OS_SYSTEM_MEMORY os_system_memory_cgroup_v2(bool query_total_ram __maybe_
195
196
done:
197
sm.ram_available_bytes = sm.ram_total_bytes - (used - inactive);
198
+ os_system_memory_last = sm;
199
return sm;
200
201
failed:
@@ -224,6 +236,7 @@ OS_SYSTEM_MEMORY os_system_memory_meminfo(bool query_total_ram __maybe_unused) {
236
}
237
238
// we keep ff open to speed up the next calls
239
+ os_system_memory_last = sm;
240
return sm;
241
242
failed:
@@ -342,6 +355,7 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {
355
356
sm.ram_available_bytes = (free_pages + inactive_pages) * page_size;
357
358
+ os_system_memory_last = sm;
359
return sm;
360
361
failed:
src/libnetdata/os/system_memory.h
+3
@@ -23,4 +23,7 @@ typedef struct {
23
// The function to get current system memory:
24
OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram);
25
26
+// Returns the last successfully reported os_system_memory() value.
27
+OS_SYSTEM_MEMORY os_last_reported_system_memory(void);
28
+
29
#endif //NETDATA_OS_MEM_AVAILABLE_H