@cryptotaxi247 / netdata-1 / commits / abd80bee1

Capture deadly signals (#19737)

* capture deadly signals on all threads; add the thread name to daemon-status-file * do not overwrite the thread * sort the exit reasons so that deadly signals are first * add -rdynamic when appropriate * log stack traces only on warnings and errors * increase status file version to 6

Costa Tsaousis committed Mar 1, 2025 at 12:52 UTC abd80bee16fdf41b6258b6726892aace20a3fff2

9 files changed +150 -86

CMakeLists.txt

+9 -1

@@ -486,6 +486,14 @@ check_function_exists(sysinfo HAVE_SYSINFO)

486

487 check_function_exists(timegm HAVE_TIMEGM)

488

489 +# Add -rdynamic when backtrace is available but libunwind is not

490 +if(HAVE_BACKTRACE AND NOT ENABLE_LIBUNWIND AND NOT STATIC_BUILD)

491 + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang")

492 + add_link_options(-rdynamic)

493 + message(STATUS "Adding -rdynamic to link options for better backtrace support")

494 + endif()

495 +endif()

496 +

497 #

498 # check source compilation

499 #

@@ -1030,7 +1038,7 @@ set(LIBNETDATA_FILES

1038 src/libnetdata/locks/benchmark.h

1039 src/libnetdata/locks/benchmark-rw.c

1040 src/libnetdata/locks/benchmark-rw.h

1033 - src/libnetdata/log/nd_log-libunwind.c

1041 + src/libnetdata/log/nd_log-stacktrace.c

1042 src/libnetdata/memory/nd-mallocz.c

1043 src/libnetdata/memory/nd-mallocz.h

1044 src/libnetdata/memory/nd-mmap.c

src/daemon/daemon-status-file.c

+43 -15

@@ -9,7 +9,7 @@

9 #include <openssl/pem.h>

10 #include <openssl/err.h>

12 -#define STATUS_FILE_VERSION 5

12 +#define STATUS_FILE_VERSION 6

14 #define STATUS_FILENAME "status-netdata.json"

@@ -152,6 +152,7 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

152 buffer_json_member_add_string_or_empty(wb, "function", ds->fatal.function);

153 buffer_json_member_add_string_or_empty(wb, "message", ds->fatal.message);

154 buffer_json_member_add_string_or_empty(wb, "errno", ds->fatal.errno_str);

155 + buffer_json_member_add_string_or_empty(wb, "thread", ds->fatal.thread);

156 buffer_json_member_add_string_or_empty(wb, "stack_trace", ds->fatal.stack_trace);

157 }

158 buffer_json_object_close(wb);

@@ -190,6 +191,7 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

191 bool required_v1 = version >= 1 ? strict : false;

192 bool required_v3 = version >= 3 ? strict : false;

193 bool required_v4 = version >= 4 ? strict : false;

194 + bool required_v5 = version >= 5 ? strict : false;

195

196 // Parse timestamp

197 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required_v1);

@@ -264,9 +266,10 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

266 JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "filename", ds->fatal.filename, error, required_v1);

267 JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "function", ds->fatal.function, error, required_v1);

268 JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "message", ds->fatal.message, error, required_v1);

267 - JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "errno", ds->fatal.errno_str, error, required_v3);

268 - JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "stack_trace", ds->fatal.stack_trace, error, required_v1);

269 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "stack_trace", ds->fatal.stack_trace, error, required_v1);

270 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "line", ds->fatal.line, error, required_v1);

271 + JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "errno", ds->fatal.errno_str, error, required_v3);

272 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "thread", ds->fatal.thread, error, required_v5);

273 });

274

275 // Parse the last posted object

@@ -536,20 +539,25 @@ static bool save_status_file(const char *directory, const char *content, size_t

539 return true;

540 }

541

542 +static BUFFER *static_save_buffer = NULL;

543 +static void static_save_buffer_init(void) {

544 + if (!static_save_buffer)

545 + static_save_buffer = buffer_create(16384, NULL);

546 +

547 + buffer_flush(static_save_buffer);

548 +}

549 +

550 static void daemon_status_file_save(DAEMON_STATUS_FILE *ds) {

551 spinlock_lock(&dsf_spinlock);

552

542 - static BUFFER *wb = NULL;

543 - if (!wb)

544 - wb = buffer_create(16384, NULL);

553 + static_save_buffer_init();

554

546 - buffer_flush(wb);

547 - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);

548 - daemon_status_file_to_json(wb, ds);

549 - buffer_json_finalize(wb);

555 + buffer_json_initialize(static_save_buffer, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);

556 + daemon_status_file_to_json(static_save_buffer, ds);

557 + buffer_json_finalize(static_save_buffer);

558

551 - const char *content = buffer_tostring(wb);

552 - size_t content_size = buffer_strlen(wb);

559 + const char *content = buffer_tostring(static_save_buffer);

560 + size_t content_size = buffer_strlen(static_save_buffer);

561

562 // Try primary directory first

563 bool saved = false;

@@ -591,6 +599,23 @@ static void daemon_status_file_out_of_memory(void) {

599 daemon_status_file_exit_reason_save(EXIT_REASON_OUT_OF_MEMORY);

600 }

601

602 +void daemon_status_file_bad_signal_received(EXIT_REASON reason) {

603 + spinlock_lock(&dsf_spinlock);

604 + session_status.exit_reason |= reason;

605 +

606 + if(!session_status.fatal.thread[0])

607 + strncpyz(session_status.fatal.thread, nd_thread_tag(), sizeof(session_status.fatal.thread) - 1);

608 +

609 + if(!session_status.fatal.stack_trace[0]) {

610 + static_save_buffer_init();

611 + capture_stack_trace(static_save_buffer);

612 + strncpyz(session_status.fatal.stack_trace, buffer_tostring(static_save_buffer), sizeof(session_status.fatal.stack_trace) - 1);

613 + }

614 +

615 + spinlock_unlock(&dsf_spinlock);

616 + daemon_status_file_save(&session_status);

617 +}

618 +

619 // --------------------------------------------------------------------------------------------------------------------

620 // deduplication hashes management

621

@@ -948,7 +973,7 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio

973 spinlock_lock(&dsf_spinlock);

974

975 // do not check the function, because it may have a startup step in it

951 - if(session_status.fatal.filename || session_status.fatal.message || session_status.fatal.errno_str || session_status.fatal.stack_trace) {

976 + if(session_status.fatal.filename || session_status.fatal.message || session_status.fatal.errno_str || session_status.fatal.thread[0]) {

977 spinlock_unlock(&dsf_spinlock);

978 freez((void *)filename);

979 freez((void *)function);

@@ -958,16 +983,19 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio

983 return;

984 }

985

986 + exit_initiated |= EXIT_REASON_FATAL;

987 + strncpyz(session_status.fatal.thread, nd_thread_tag(), sizeof(session_status.fatal.thread) - 1);

988 +

989 session_status.fatal.filename = filename;

990 freez((char *)session_status.fatal.function); // it may have a startup step

991 session_status.fatal.function = function;

992 session_status.fatal.message = message;

993 session_status.fatal.errno_str = errno_str;

966 - session_status.fatal.stack_trace = stack_trace;

994 + strncpyz(session_status.fatal.stack_trace, stack_trace, sizeof(session_status.fatal.stack_trace) - 1);

995 + freez((char *)stack_trace);

996 session_status.fatal.line = line;

997

998 spinlock_unlock(&dsf_spinlock);

999

971 - exit_initiated |= EXIT_REASON_FATAL;

1000 daemon_status_file_save(&session_status);

1001 }

src/daemon/daemon-status-file.h

+3 -1

@@ -66,8 +66,9 @@ typedef struct daemon_status_file {

66 const char *filename;

67 const char *function;

68 const char *errno_str;

69 - const char *stack_trace;

69 const char *message;

70 + char stack_trace[4096];

71 + char thread[ND_THREAD_TAG_MAX + 1];

72 } fatal;

74 struct {

@@ -82,6 +83,7 @@ DAEMON_STATUS_FILE daemon_status_file_load(void);

83 // saves the current status

84 void daemon_status_file_update_status(DAEMON_STATUS status);

85 void daemon_status_file_exit_reason_save(EXIT_REASON reason);

86 +void daemon_status_file_bad_signal_received(EXIT_REASON reason);

88 // check for a crash

89 void daemon_status_file_check_crash(void);

src/daemon/signals.c

+18 -7

@@ -42,18 +42,22 @@ static void signal_handler(int signo) {

42 }

44 int i;

45 - for(i = 0; signals_waiting[i].action != NETDATA_SIGNAL_END_OF_LIST ; i++) {

45 + for(i = 0; signals_waiting[i].action != NETDATA_SIGNAL_END_OF_LIST; i++) {

46 if(unlikely(signals_waiting[i].signo == signo)) {

47 signals_waiting[i].count++;

49 if(signals_waiting[i].action == NETDATA_SIGNAL_FATAL) {

50 char buffer[200 + 1];

51 - snprintfz(buffer, sizeof(buffer) - 1, "\nSIGNAL HANDLER: received: %s. Oops! This is bad!\n", signals_waiting[i].name);

51 + snprintfz(buffer, sizeof(buffer) - 1, "\nSIGNAL HANDLER: received: %s in thread %d. Oops! This is bad!\n",

52 + signals_waiting[i].name, gettid_cached());

53 +

54 if(write(STDERR_FILENO, buffer, strlen(buffer)) == -1) {

55 // nothing to do - we cannot write but there is no way to complain about it

56 ;

57 }

56 - daemon_status_file_exit_reason_save(signals_waiting[i].reason);

58 +

59 + // Always update the status file for fatal signals

60 + daemon_status_file_bad_signal_received(signals_waiting[i].reason);

61 }

63 break;

@@ -63,15 +67,22 @@ static void signal_handler(int signo) {

67 __atomic_sub_fetch(&recurse, 1, __ATOMIC_RELAXED);

68 }

70 +

71 // Mask all signals, to ensure they will only be unmasked at the threads that can handle them.

72 // This means that all third party libraries (including libuv) cannot use signals anymore.

73 // The signals they are interested must be unblocked at their corresponding event loops.

69 -static void posix_mask_all_signals(void) {

74 +static void posix_signals_default_mask(void) {

75 sigset_t sigset;

76 sigfillset(&sigset);

78 + // Don't mask fatal signals - we want these to be handled in any thread

79 + sigdelset(&sigset, SIGBUS);

80 + sigdelset(&sigset, SIGSEGV);

81 + sigdelset(&sigset, SIGFPE);

82 + sigdelset(&sigset, SIGILL);

83 +

84 if(pthread_sigmask(SIG_BLOCK, &sigset, NULL) != 0)

74 - netdata_log_error("SIGNAL: cannot mask all signals");

85 + netdata_log_error("SIGNAL: cannot apply the default mask for signals");

86 }

88 // Unmask all signals the netdata main signal handler uses.

@@ -88,7 +99,7 @@ static void posix_unmask_my_signals(void) {

99 }

100

101 void nd_initialize_signals(void) {

91 - posix_mask_all_signals(); // block all signals for all threads

102 + posix_signals_default_mask();

103

104 // Catch signals which we want to use

105 struct sigaction sa;

@@ -164,7 +175,7 @@ void nd_process_signals(void) {

175

176 case NETDATA_SIGNAL_FATAL:

177 nd_log_limits_unlimited();

167 - daemon_status_file_exit_reason_save(signals_waiting[i].reason);

178 + daemon_status_file_bad_signal_received(signals_waiting[i].reason);

179 fatal("SIGNAL: Received %s. netdata now exits.", name);

180 break;

181

src/libnetdata/exit/exit_initiated.h

+19 -19

@@ -9,31 +9,31 @@

9 typedef enum {

10 EXIT_REASON_NONE = 0,

12 - // automatically detect when exit_initiated_set() is called

13 - // supports Linux, FreeBSD, MacOS, Windows

14 - EXIT_REASON_SYSTEM_SHUTDOWN = (1 << 0), // detected

15 -

16 - // signals - normal termination

17 - EXIT_REASON_SIGQUIT = (1 << 1), // rare, but graceful

18 - EXIT_REASON_SIGTERM = (1 << 2), // received on Linux, FreeBSD, MacOS

19 - EXIT_REASON_SIGINT = (1 << 3), // received on Windows on normal termination

20 -

12 // signals - abnormal termination

22 - EXIT_REASON_SIGBUS = (1 << 4),

23 - EXIT_REASON_SIGSEGV = (1 << 5),

24 - EXIT_REASON_SIGFPE = (1 << 6),

25 - EXIT_REASON_SIGILL = (1 << 7),

26 - EXIT_REASON_OUT_OF_MEMORY = (1 << 8),

13 + EXIT_REASON_SIGBUS = (1 << 0),

14 + EXIT_REASON_SIGSEGV = (1 << 1),

15 + EXIT_REASON_SIGFPE = (1 << 2),

16 + EXIT_REASON_SIGILL = (1 << 3),

17 + EXIT_REASON_OUT_OF_MEMORY = (1 << 4),

18 +

19 + // abnormal termination via a fatal message

20 + EXIT_REASON_FATAL = (1 << 5), // a fatal message

22 // normal termination via APIs

29 - EXIT_REASON_API_QUIT = (1 << 9),

30 - EXIT_REASON_CMD_EXIT = (1 << 10),

23 + EXIT_REASON_API_QUIT = (1 << 6), // developer only

24 + EXIT_REASON_CMD_EXIT = (1 << 7), // netdatacli

32 - // abnormal termination via a fatal message

33 - EXIT_REASON_FATAL = (1 << 11),

26 + // signals - normal termination

27 + EXIT_REASON_SIGQUIT = (1 << 8), // rare, but graceful

28 + EXIT_REASON_SIGTERM = (1 << 9), // received on Linux, FreeBSD, MacOS

29 + EXIT_REASON_SIGINT = (1 << 10), // received on Windows on normal termination

31 // windows specific, service stop

36 - EXIT_REASON_SERVICE_STOP = (1 << 12),

32 + EXIT_REASON_SERVICE_STOP = (1 << 11),

33 +

34 + // automatically detect when exit_initiated_set() is called

35 + // supports Linux, FreeBSD, MacOS, Windows

36 + EXIT_REASON_SYSTEM_SHUTDOWN = (1 << 12),

38 // netdata update

39 EXIT_REASON_UPDATE = (1 << 13),

src/libnetdata/log/nd_log-init.c

@@ -281,6 +281,8 @@ int nd_log_systemd_journal_fd(void) {

281

282 void nd_log_reopen_log_files_for_spawn_server(const char *name) {

283 nd_log_forked = true;

284 + nd_log.log_event_cb = NULL;

285 + nd_log.fatal_event_cb = NULL;

286

287 gettid_uncached();

288

src/libnetdata/log/nd_log-stacktrace.c renamed

+54 -29

@@ -4,26 +4,10 @@

5 bool nd_log_forked = false;

7 -#ifdef HAVE_LIBUNWIND

7 +#if defined(HAVE_LIBUNWIND)

8 #include <libunwind.h>

10 -bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

11 - static __thread bool in_stack_trace = false;

12 -

13 - if (nd_log_forked) {

14 - // libunwind freezes in forked children

15 - buffer_strcat(wb, "stack trace after fork is disabled");

16 - return true;

17 - }

18 -

19 - if (in_stack_trace) {

20 - // Prevent recursion

21 - buffer_strcat(wb, "stack trace recursion detected");

22 - return true;

23 - }

24 -

25 - in_stack_trace = true;

26 -

10 +void capture_stack_trace(BUFFER *wb) {

11 unw_cursor_t cursor;

12 unw_context_t context;

13 size_t frames = 0;

@@ -32,11 +16,11 @@ bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

16 unw_getcontext(&context);

17 unw_init_local(&cursor, &context);

35 -// // Skip first 3 frames (our logging infrastructure)

36 -// for (int i = 0; i < 3; i++) {

37 -// if (unw_step(&cursor) <= 0)

38 -// goto cleanup; // Ensure proper cleanup if unwinding fails early

39 -// }

19 + // // Skip first 3 frames (our logging infrastructure)

20 + // for (int i = 0; i < 3; i++) {

21 + // if (unw_step(&cursor) <= 0)

22 + // goto cleanup; // Ensure proper cleanup if unwinding fails early

23 + // }

25 while (unw_step(&cursor) > 0) {

26 unw_word_t offset, pc;

@@ -57,16 +41,57 @@ bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

41 buffer_strcat(wb, "<unknown>");

42 }

43 }

44 +}

61 - in_stack_trace = false; // Ensure the flag is reset

62 - return true;

46 +#elif defined(HAVE_BACKTRACE)

47 +

48 +void capture_stack_trace(BUFFER *wb) {

49 + void *array[50];

50 + char **messages;

51 + int size, i;

52 +

53 + size = backtrace(array, _countof(array));

54 + messages = backtrace_symbols(array, size);

55 +

56 + if (messages == NULL) {

57 + // Failed to get backtrace symbols

58 + return;

59 + }

60 +

61 + // Format the stack trace

62 + for (i = 0; i < size ; i++)

63 + buffer_sprintf(wb, "#%d %s\n", i, messages[i]);

64 +

65 + free(messages);

66 }

65 -#else // !HAVE_LIBUNWIND

68 +#else

67 -bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

68 - buffer_strcat(wb, "libunwind not available");

69 - return true;

70 +void capture_stack_trace(BUFFER *wb) {

71 + buffer_strcat(wb, "stack trace not available");

72 }

74 #endif

75 +

76 +bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

77 + static __thread bool in_stack_trace = false;

78 +

79 + if (nd_log_forked) {

80 + // libunwind freezes in forked children

81 + buffer_strcat(wb, "stack trace after fork is disabled");

82 + return true;

83 + }

84 +

85 + if (in_stack_trace) {

86 + // Prevent recursion

87 + buffer_strcat(wb, "stack trace recursion detected");

88 + return true;

89 + }

90 +

91 + in_stack_trace = true;

92 +

93 + capture_stack_trace(wb);

94 +

95 + in_stack_trace = false; // Ensure the flag is reset

96 + return true;

97 +}

src/libnetdata/log/nd_log.c

+1 -14

@@ -268,7 +268,7 @@ static void nd_logger(const char *file, const char *function, const unsigned lon

268

269 // set the common fields that are automatically set by the logging subsystem

270

271 - if(likely(!thread_log_fields[NDF_STACK_TRACE].entry.set))

271 + if(likely(!thread_log_fields[NDF_STACK_TRACE].entry.set) && priority <= NDLP_WARNING)

272 thread_log_fields[NDF_STACK_TRACE].entry = ND_LOG_FIELD_CB(NDF_STACK_TRACE, stack_trace_formatter, NULL);

273

274 if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))

@@ -517,19 +517,6 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned

517 char action_result[200+1];

518 snprintfz(action_result, 60, "%s:%s:%s", program_name, tag_to_send, function);

519

520 -#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)

521 - int fd = nd_log.sources[NDLS_DAEMON].fd;

522 - if(fd == -1)

523 - fd = STDERR_FILENO;

524 -

525 - int nptrs;

526 - void *buffer[10000];

527 -

528 - nptrs = backtrace(buffer, sizeof(buffer));

529 - if(nptrs)

530 - backtrace_symbols_fd(buffer, nptrs, fd);

531 -#endif

532 -

520 #ifdef NETDATA_INTERNAL_CHECKS

521 // abort();

522 #endif

src/libnetdata/log/nd_log.h

@@ -32,6 +32,7 @@ int nd_log_priority2id(const char *priority);

32 const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);

33 const char *nd_log_method_for_external_plugins(const char *s);

34 ND_UUID nd_log_get_invocation_id(void);

35 +void capture_stack_trace(BUFFER *wb);

37 typedef void (*log_event_t)(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);

38 void nd_log_register_event_cb(log_event_t cb);