save status on out of memory event (#19719)
Costa Tsaousis committed
Feb 26, 2025 at 15:43 UTC
ddbf7003cccc67fdb9e326880d7e7ca770090e1e
4 files changed
+45
-23
src/daemon/daemon-status-file.c
+12
-2
@@ -498,8 +498,10 @@ static bool save_status_file(const char *directory, const char *content, size_t
498
static void daemon_status_file_save(DAEMON_STATUS_FILE *ds) {
499
spinlock_lock(&dsf_spinlock);
500
501
- // Prepare JSON content
502
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
501
+ static BUFFER *wb = NULL;
502
+ if (!wb)
503
+ wb = buffer_create(16384, NULL);
504
+
505
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
506
daemon_status_file_to_json(wb, ds);
507
buffer_json_finalize(wb);
@@ -543,6 +545,10 @@ void daemon_status_file_exit_reason_save(EXIT_REASON reason) {
545
daemon_status_file_save(&session_status);
546
}
547
548
+static void daemon_status_file_out_of_memory(void) {
549
+ daemon_status_file_exit_reason_save(EXIT_REASON_OUT_OF_MEMORY);
550
+}
551
+
552
// --------------------------------------------------------------------------------------------------------------------
553
// POST the last status to agent-events
554
@@ -602,6 +608,10 @@ void *post_status_file_thread(void *ptr) {
608
// check last status on startup and post-crash report
609
610
void daemon_status_file_check_crash(void) {
611
+ FUNCTION_RUN_ONCE();
612
+
613
+ mallocz_register_out_of_memory_cb(daemon_status_file_out_of_memory);
614
+
615
last_session_status = daemon_status_file_load();
616
daemon_status_file_update_status(DAEMON_STATUS_INITIALIZING);
617
ND_LOG_FIELD_PRIORITY pri = NDLP_NOTICE;
src/libnetdata/exit/exit_initiated.h
+7
-6
@@ -23,23 +23,24 @@ typedef enum {
23
EXIT_REASON_SIGSEGV = (1 << 5),
24
EXIT_REASON_SIGFPE = (1 << 6),
25
EXIT_REASON_SIGILL = (1 << 7),
26
+ EXIT_REASON_OUT_OF_MEMORY = (1 << 8),
27
28
// normal termination via APIs
28
- EXIT_REASON_API_QUIT = (1 << 7),
29
- EXIT_REASON_CMD_EXIT = (1 << 8),
29
+ EXIT_REASON_API_QUIT = (1 << 9),
30
+ EXIT_REASON_CMD_EXIT = (1 << 10),
31
32
// abnormal termination via a fatal message
32
- EXIT_REASON_FATAL = (1 << 9),
33
+ EXIT_REASON_FATAL = (1 << 11),
34
35
// windows specific, service stop
35
- EXIT_REASON_SERVICE_STOP = (1 << 10),
36
+ EXIT_REASON_SERVICE_STOP = (1 << 12),
37
38
// netdata update
38
- EXIT_REASON_UPDATE = (1 << 11),
39
+ EXIT_REASON_UPDATE = (1 << 13),
40
} EXIT_REASON;
41
42
#define EXIT_REASON_NORMAL (EXIT_REASON_SIGINT|EXIT_REASON_SIGTERM|EXIT_REASON_SIGQUIT|EXIT_REASON_API_QUIT|EXIT_REASON_CMD_EXIT|EXIT_REASON_SERVICE_STOP|EXIT_REASON_SYSTEM_SHUTDOWN|EXIT_REASON_UPDATE)
42
-#define EXIT_REASON_ABNORMAL (EXIT_REASON_SIGBUS|EXIT_REASON_SIGSEGV|EXIT_REASON_SIGFPE|EXIT_REASON_SIGILL|EXIT_REASON_FATAL)
43
+#define EXIT_REASON_ABNORMAL (EXIT_REASON_SIGBUS|EXIT_REASON_SIGSEGV|EXIT_REASON_SIGFPE|EXIT_REASON_SIGILL|EXIT_REASON_FATAL|EXIT_REASON_OUT_OF_MEMORY)
44
45
#define is_exit_reason_normal(reason) (((reason) & EXIT_REASON_NORMAL) && !((reason) & EXIT_REASON_ABNORMAL))
46
src/libnetdata/memory/nd-mallocz.c
+23
-15
@@ -2,6 +2,29 @@
2
3
#include "../libnetdata.h"
4
5
+out_of_memory_cb out_of_memory_callback = NULL;
6
+void mallocz_register_out_of_memory_cb(out_of_memory_cb cb) {
7
+ out_of_memory_callback = cb;
8
+}
9
+
10
+ALWAYS_INLINE NORETURN
11
+static void out_of_memory(const char *call, size_t size) {
12
+ if(out_of_memory_callback)
13
+ out_of_memory_callback();
14
+
15
+ struct rusage usage = { 0 };
16
+ if(getrusage(RUSAGE_SELF, &usage) != 0)
17
+ usage.ru_maxrss = 0;
18
+
19
+ OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
20
+ fatal("Out of memory on %s(%zu bytes)!\n"
21
+ "System memory available: %lu, while our max RSS usage is: %ld\n"
22
+ "O/S mmap limit: %llu, while our mmap count is: %zu",
23
+ call, size,
24
+ sm.ram_available_bytes, usage.ru_maxrss,
25
+ os_mmap_limit(), __atomic_load_n(&nd_mmap_count, __ATOMIC_RELAXED));
26
+}
27
+
28
// ----------------------------------------------------------------------------
29
// memory allocation functions that handle failures
30
@@ -386,21 +409,6 @@ void freez_int(void *ptr, const char *file, const char *function, size_t line) {
409
}
410
#else
411
389
-ALWAYS_INLINE NORETURN
390
-static void out_of_memory(const char *call, size_t size) {
391
- struct rusage usage = { 0 };
392
- if(getrusage(RUSAGE_SELF, &usage) != 0)
393
- usage.ru_maxrss = 0;
394
-
395
- OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
396
- fatal("Out of memory on %s(%zu bytes)!\n"
397
- "System memory available: %lu, while our max RSS usage is: %ld\n"
398
- "O/S mmap limit: %llu, while our mmap count is: %zu",
399
- call, size,
400
- sm.ram_available_bytes, usage.ru_maxrss,
401
- os_mmap_limit(), __atomic_load_n(&nd_mmap_count, __ATOMIC_RELAXED));
402
-}
403
-
412
ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED
413
char *strdupz(const char *s) {
414
workers_memory_call(WORKERS_MEMORY_CALL_LIBC_STRDUP);
src/libnetdata/memory/nd-mallocz.h
+3
@@ -64,4 +64,7 @@ void mallocz_release_as_much_memory_to_the_system(void);
64
int posix_memalignz(void **memptr, size_t alignment, size_t size);
65
void posix_memalign_freez(void *ptr);
66
67
+typedef void (*out_of_memory_cb)(void);
68
+void mallocz_register_out_of_memory_cb(out_of_memory_cb cb);
69
+
70
#endif //NETDATA_ND_MALLOCZ_H