@cryptotaxi247 / netdata-1 / commits / 568766f2d

daemon status 26d (#20047)

* filter signal handler from stack traces When capturing stack traces from crashes where signal handlers are involved, filter out the signal handler function itself and all functions after it. This provides cleaner and more focused stack traces by showing only the code path leading up to the crash. * improve stack trace processing for crashes 1. Add root cause tracking to store the first netdata source function in a stack trace - only for libbacktrace 2. Update status-file.c to use the root cause function for crash reporting * mark functions as never inline to ensure they are always in the stack traces * show the logger function in the stack traces * never skip frames from stack traces

Costa Tsaousis committed Apr 3, 2025 at 17:12 UTC 568766f2daec5b85f96de6a3b816af6cde788d39

5 files changed +205 -6

src/daemon/signal-handler.c

@@ -47,6 +47,7 @@ static struct {

47 static void (*original_handlers[NSIG])(int) = {0};

48 static void (*original_sigactions[NSIG])(int, siginfo_t *, void *) = {0};

50 +NEVER_INLINE

51 void nd_signal_handler(int signo, siginfo_t *info, void *context __maybe_unused) {

53 for(size_t i = 0; i < _countof(signals_waiting) ; i++) {

@@ -168,6 +169,9 @@ void nd_cleanup_deadly_signals(void) {

169

170 void nd_initialize_signals(bool chain_existing) {

171 signals_block_all_except_deadly();

172 +

173 + // Set the signal handler name for stack trace filtering

174 + capture_stack_trace_set_signal_handler_function("nd_signal_handler");

175

176 struct sigaction act;

177 memset(&act, 0, sizeof(struct sigaction));

@@ -206,6 +210,7 @@ void nd_initialize_signals(bool chain_existing) {

210 }

211 }

212

213 +NEVER_INLINE

214 static void process_triggered_signals(void) {

215 size_t found;

216 do {

@@ -264,6 +269,7 @@ static inline bool threshold_trigger_smaller(bool *last, double threshold, doubl

269 return !triggered && *last;

270 }

271

272 +NEVER_INLINE

273 void nd_process_signals(void) {

274 posix_unmask_my_signals();

275 const usec_t save_every_ut = 15 * 60 * USEC_PER_SEC;

src/daemon/status-file.c

+10

@@ -1140,6 +1140,7 @@ void daemon_status_file_check_crash(void) {

1140 }

1141 }

1142

1143 +NEVER_INLINE

1144 static void daemon_status_file_save_twice_if_we_can_get_stack_trace(BUFFER *wb, DAEMON_STATUS_FILE *ds, bool force) {

1145 // IMPORTANT: NO LOCKS OR ALLOCATIONS HERE, THIS FUNCTION IS CALLED FROM SIGNAL HANDLERS

1146 // THIS FUNCTION MUST USE ONLY ASYNC-SIGNAL-SAFE OPERATIONS

@@ -1156,8 +1157,14 @@ static void daemon_status_file_save_twice_if_we_can_get_stack_trace(BUFFER *wb,

1157 return;

1158

1159 buffer_flush(wb);

1160 +

1161 capture_stack_trace(wb);

1162

1163 + // Store the first netdata function from the stack trace if available

1164 + const char *first_nd_fn = capture_stack_trace_root_cause_function();

1165 + if (first_nd_fn && *first_nd_fn && !ds->fatal.function[0])

1166 + strncpyz(ds->fatal.function, first_nd_fn, sizeof(ds->fatal.function) - 1);

1167 +

1168 if(buffer_strlen(wb) > 0) {

1169 strncpyz(

1170 ds->fatal.stack_trace,

@@ -1173,6 +1180,7 @@ static void daemon_status_file_save_twice_if_we_can_get_stack_trace(BUFFER *wb,

1180 // --------------------------------------------------------------------------------------------------------------------

1181 // ng_log() hook for receiving fatal message information

1182

1183 +NEVER_INLINE

1184 void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line) {

1185 FUNCTION_RUN_ONCE();

1186

@@ -1251,6 +1259,7 @@ static void daemon_status_file_out_of_memory(void) {

1259 daemon_status_file_save_twice_if_we_can_get_stack_trace(static_save_buffer, &session_status, true);

1260 }

1261

1262 +NEVER_INLINE

1263 bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE code, void *fault_address, bool chained_handler) {

1264 FUNCTION_RUN_ONCE_RET(true);

1265

@@ -1316,6 +1325,7 @@ bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE c

1325

1326 static SPINLOCK shutdown_timeout_spinlock = SPINLOCK_INITIALIZER;

1327

1328 +NEVER_INLINE

1329 void daemon_status_file_shutdown_timeout(BUFFER *trace) {

1330 FUNCTION_RUN_ONCE();

1331

src/libnetdata/log/nd_log-stacktrace.c

+184 -6

@@ -6,6 +6,109 @@ bool nd_log_forked = false;

7 #define NO_STACK_TRACE_PREFIX "info: stack trace is not available, "

9 +// The signal handler function name to filter out in stack traces

10 +static const char *signal_handler_function = "nd_signal_handler";

11 +

12 +// List of auxiliary functions that should not be reported as root cause

13 +static const char *auxiliary_functions[] = {

14 + "nd_uuid_copy",

15 + "out_of_memory",

16 + "shutdown_timed_out",

17 + NULL // Terminator

18 +};

19 +

20 +// List of logging functions to filter out

21 +static const char *logging_functions[] = {

22 + "netdata_logger",

23 + "netdata_logger_with_limit",

24 + "netdata_logger_fatal",

25 + NULL // Terminator

26 +};

27 +

28 +// Set the signal handler function name to filter out in stack traces

29 +void capture_stack_trace_set_signal_handler_function(const char *function_name) {

30 + signal_handler_function = function_name;

31 +}

32 +

33 +// Exact match check if a function is in the auxiliary list (strcmp)

34 +static inline bool is_auxiliary_function(const char *function) {

35 + if (!function || !*function)

36 + return false;

37 +

38 + for (int i = 0; auxiliary_functions[i]; i++) {

39 + if (strcmp(function, auxiliary_functions[i]) == 0)

40 + return true;

41 + }

42 +

43 + return false;

44 +}

45 +

46 +// Exact match check if a function is a logging function (strcmp)

47 +static inline bool is_logging_function(const char *function) {

48 + if (!function || !*function)

49 + return false;

50 +

51 + for (int i = 0; logging_functions[i]; i++) {

52 + if (strcmp(function, logging_functions[i]) == 0)

53 + return true;

54 + }

55 +

56 + return false;

57 +}

58 +

59 +// Substring check if a function contains a logging function name (strstr)

60 +static inline bool contains_logging_function(const char *text) {

61 + if (!text || !*text)

62 + return false;

63 +

64 + for (int i = 0; logging_functions[i]; i++) {

65 + if (strstr(text, logging_functions[i]) != NULL)

66 + return true;

67 + }

68 +

69 + return false;

70 +}

71 +

72 +static inline bool is_netdata_function(const char *function, const char *filename) {

73 + return function && *function && filename && *filename &&

74 + strstr(filename, "/src/") &&

75 + !strstr(filename, "/vendored/");

76 +}

77 +

78 +// Exact match check if a function is the signal handler (strcmp)

79 +static inline bool is_signal_handler_function(const char *function) {

80 + return function && *function &&

81 + signal_handler_function && *signal_handler_function &&

82 + strcmp(function, signal_handler_function) == 0;

83 +}

84 +

85 +// Substring check if a function contains the signal handler name (strstr)

86 +static inline bool contains_signal_handler_function(const char *text) {

87 + return text && *text &&

88 + signal_handler_function && *signal_handler_function &&

89 + strstr(text, signal_handler_function) != NULL;

90 +}

91 +

92 +// Thread-local buffer to store the first netdata function encountered in a stack trace

93 +static __thread char root_cause_function[48];

94 +

95 +// Returns the first netdata function found in the stack trace

96 +const char *capture_stack_trace_root_cause_function(void) {

97 + return root_cause_function[0] ? root_cause_function : NULL;

98 +}

99 +

100 +// Store a function name as the first netdata function found

101 +static inline void keep_first_root_cause_function(const char *function) {

102 + if (!function || !*function || root_cause_function[0])

103 + return; // Already have a function or null input

104 +

105 + // Skip auxiliary functions and logging functions

106 + if (is_auxiliary_function(function) || is_logging_function(function))

107 + return;

108 +

109 + strncpyz(root_cause_function, function, sizeof(root_cause_function) - 1);

110 +}

111 +

112 #if defined(HAVE_LIBBACKTRACE)

113 #include "backtrace-supported.h"

114 #endif

@@ -16,9 +119,10 @@ bool nd_log_forked = false;

119 static struct backtrace_state *backtrace_state = NULL;

120

121 typedef struct {

19 - BUFFER *wb; // Buffer to write to

20 - size_t frame_count; // Number of frames processed

21 - bool first_frame; // Is this the first frame?

122 + BUFFER *wb; // Buffer to write to

123 + size_t frame_count; // Number of frames processed

124 + bool first_frame; // Is this the first frame?

125 + bool found_signal_handler; // Have we found the signal handler frame?

126 } backtrace_data_t;

127

128 // Common function to format and add a stack frame to the buffer

@@ -29,6 +133,33 @@ static void add_stack_frame(backtrace_data_t *bt_data, uintptr_t pc, const char

133 if (!wb)

134 return;

135

136 + // Check if we found the signal handler frame

137 + if (!bt_data->found_signal_handler && is_signal_handler_function(function)) {

138 + // We found the signal handler, reset the buffer and clear function name

139 + buffer_flush(wb);

140 + bt_data->frame_count = 0;

141 + bt_data->first_frame = true;

142 + bt_data->found_signal_handler = true;

143 + root_cause_function[0] = '\0';

144 + return; // Skip adding the signal handler itself

145 + }

146 +

147 + // Check for logging functions, but only if we haven't found a signal handler yet

148 + // This prevents double resets when crashing inside logging code

149 + if (!bt_data->found_signal_handler && is_logging_function(function)) {

150 + // Found a logging function, reset the buffer and clear function name

151 + buffer_flush(wb);

152 + bt_data->frame_count = 0;

153 + bt_data->first_frame = true;

154 + root_cause_function[0] = '\0';

155 + // continue to add the function to the stack trace

156 + }

157 +

158 + // Check if this is a netdata source file and store the function name if it is

159 + // (but only if we haven't already stored one)

160 + if (!root_cause_function[0] && is_netdata_function(function, filename))

161 + keep_first_root_cause_function(function);

162 +

163 // Add a newline between frames

164 if (!bt_data->first_frame)

165 buffer_putc(wb, '\n');

@@ -164,6 +295,7 @@ bool capture_stack_trace_available(void) {

295 return backtrace_state != NULL && BACKTRACE_SUPPORTED;

296 }

297

298 +NEVER_INLINE

299 void capture_stack_trace(BUFFER *wb) {

300 if (!backtrace_state) {

301 buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libbacktrace not initialized");

@@ -173,11 +305,12 @@ void capture_stack_trace(BUFFER *wb) {

305 backtrace_data_t bt_data = {

306 .wb = wb,

307 .frame_count = 0,

176 - .first_frame = true

308 + .first_frame = true,

309 + .found_signal_handler = false

310 };

311

312 // Skip one frame to hide capture_stack_trace() itself

180 - backtrace_full(backtrace_state, 1, bt_full_handler,

313 + backtrace_full(backtrace_state, 0, bt_full_handler,

314 bt_error_handler, &bt_data);

315

316 // If no frames were reported

@@ -216,6 +349,7 @@ bool capture_stack_trace_available(void) {

349 return true;

350 }

351

352 +NEVER_INLINE

353 void capture_stack_trace(BUFFER *wb) {

354 // this function is async-signal-safe, if the buffer has enough space to hold the stack trace

355

@@ -228,6 +362,8 @@ void capture_stack_trace(BUFFER *wb) {

362 unw_init_local(&cursor, &context);

363

364 size_t added = 0;

365 + bool found_signal_handler = false;

366 +

367 while (unw_step(&cursor) > 0) {

368 unw_word_t offset, pc;

369 char sym[256];

@@ -242,6 +378,25 @@ void capture_stack_trace(BUFFER *wb) {

378 offset = 0;

379 }

380

381 + // Check if we found the signal handler frame

382 + if (!found_signal_handler && is_signal_handler_function(name)) {

383 + // We found the signal handler, reset the buffer

384 + buffer_flush(wb);

385 + added = 0;

386 + frames = 0;

387 + found_signal_handler = true;

388 + continue; // Skip adding the signal handler itself

389 + }

390 +

391 + // Check for logging functions, but only if we haven't found a signal handler yet

392 + if (!found_signal_handler && is_logging_function(name)) {

393 + // Found a logging function, reset the buffer

394 + buffer_flush(wb);

395 + added = 0;

396 + frames = 0;

397 + // continue to add the function to the stack trace

398 + }

399 +

400 if (frames++)

401 buffer_putc(wb, '\n');

402

@@ -284,6 +439,7 @@ bool capture_stack_trace_is_async_signal_safe(void) {

439 return false;

440 }

441

442 +NEVER_INLINE

443 void capture_stack_trace(BUFFER *wb) {

444 void *array[50];

445 char **messages;

@@ -298,14 +454,33 @@ void capture_stack_trace(BUFFER *wb) {

454 }

455

456 size_t added = 0;

457 + bool found_signal_handler = false;

458 +

459 // Format the stack trace (removing the address part)

460 for (i = 0; i < size; i++) {

461 if(messages[i] && *messages[i]) {

462 + // Check if we found the signal handler frame

463 + if (!found_signal_handler && contains_signal_handler_function(messages[i])) {

464 + // We found the signal handler, reset the buffer

465 + buffer_flush(wb);

466 + added = 0;

467 + found_signal_handler = true;

468 + continue; // Skip adding the signal handler itself

469 + }

470 +

471 + // Check for logging functions, but only if we haven't found a signal handler yet

472 + if (!found_signal_handler && contains_logging_function(messages[i])) {

473 + // Found a logging function, reset the buffer

474 + buffer_flush(wb);

475 + added = 0;

476 + // continue to add the function to the stack trace

477 + }

478 +

479 if(added)

480 buffer_putc(wb, '\n');

481

482 buffer_putc(wb, '#');

308 - buffer_print_uint64(wb, i);

483 + buffer_print_uint64(wb, added);

484 buffer_putc(wb, ' ');

485 buffer_strcat(wb, messages[i]);

486 added++;

@@ -340,6 +515,7 @@ bool capture_stack_trace_is_async_signal_safe(void) {

515 return false;

516 }

517

518 +NEVER_INLINE

519 void capture_stack_trace(BUFFER *wb) {

520 buffer_strcat(wb, NO_STACK_TRACE_PREFIX "no back-end available");

521

@@ -350,6 +526,7 @@ void capture_stack_trace(BUFFER *wb) {

526

527 #endif

528

529 +NEVER_INLINE

530 bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

531 static __thread bool in_stack_trace = false;

532

@@ -367,6 +544,7 @@ bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

544

545 in_stack_trace = true;

546

547 + root_cause_function[0] = '\0';

548 capture_stack_trace(wb);

549

550 in_stack_trace = false; // Ensure the flag is reset

src/libnetdata/log/nd_log.c

@@ -402,6 +402,7 @@ static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {

402 // --------------------------------------------------------------------------------------------------------------------

403 // public API for loggers

404

405 +NEVER_INLINE

406 void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... )

407 {

408 int saved_errno = errno;

@@ -424,6 +425,7 @@ void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const

425 va_end(args);

426 }

427

428 +NEVER_INLINE

429 void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {

430 int saved_errno = errno;

431

@@ -477,6 +479,7 @@ static void fatal_abort_internal_checks(void) {

479 _exit(1);

480 }

481

482 +NEVER_INLINE

483 void netdata_logger_fatal(const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {

484 static size_t already_in_fatal = 0;

485

src/libnetdata/log/nd_log.h

@@ -39,6 +39,8 @@ void capture_stack_trace_flush(void);

39 bool capture_stack_trace_available(void);

40 bool capture_stack_trace_is_async_signal_safe(void);

41 const char *capture_stack_trace_backend(void);

42 +void capture_stack_trace_set_signal_handler_function(const char *function_name);

43 +const char *capture_stack_trace_root_cause_function(void);

45 typedef void (*log_event_t)(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);

46 void nd_log_register_fatal_hook_cb(log_event_t cb);