@cryptotaxi247 / netdata-1 / commits / 3d5575dee

async-signal-safe stack traces (#19802)

* make stack traces unique on macos * use local unwinding only; make sure libunwind stack traces are async-signal-safe * add missing libraries when no stack traces backend is available * make sure all signal handler calls are async safe * save a status file every 10 minutes * save a status file every 15 minutes - better implementation * use monotonic clock to ensure fixed step in saving updates * simpler version

Costa Tsaousis committed Mar 8, 2025 at 13:01 UTC 3d5575dee596077256bb6a46e388315814b5937c

8 files changed +175 -76

src/daemon/daemon-status-file.c

+6 -5

@@ -9,7 +9,7 @@

9 #include <openssl/pem.h>

10 #include <openssl/err.h>

12 -#define STATUS_FILE_VERSION 11

12 +#define STATUS_FILE_VERSION 12

14 #define STATUS_FILENAME "status-netdata.json"

@@ -1119,7 +1119,7 @@ void daemon_status_file_update_status(DAEMON_STATUS status) {

1119 static void daemon_status_file_out_of_memory(void) {

1120 FUNCTION_RUN_ONCE();

1121

1122 - // DO NOT LOCK OR ALLOCATE IN THIS FUNCTION - WE DON'T HAVE ANY MEMORY AVAILABLE - IT HAPPENED ALREADY!

1122 + // DO NOT ALLOCATE IN THIS FUNCTION - WE DON'T HAVE ANY MEMORY AVAILABLE!

1123

1124 exit_initiated_add(EXIT_REASON_OUT_OF_MEMORY);

1125

@@ -1142,7 +1142,7 @@ void daemon_status_file_deadly_signal_received(EXIT_REASON reason) {

1142

1143 session_status.exit_reason |= reason;

1144 if(!session_status.fatal.thread[0])

1145 - strncpyz(session_status.fatal.thread, nd_thread_tag(), sizeof(session_status.fatal.thread) - 1);

1145 + strncpyz(session_status.fatal.thread, nd_thread_tag_async_safe(), sizeof(session_status.fatal.thread) - 1);

1146

1147 dsf_release(session_status);

1148

@@ -1152,8 +1152,9 @@ void daemon_status_file_deadly_signal_received(EXIT_REASON reason) {

1152 // save what we know already

1153 daemon_status_file_save(static_save_buffer, &session_status, false);

1154

1155 - // we cannot get a stack trace on SIGABRT - it may deadlock forever

1156 - if(reason != EXIT_REASON_SIGABRT && !session_status.fatal.stack_trace[0]) {

1155 + bool can_safely_capture_stack_trace = reason != EXIT_REASON_SIGABRT || capture_stack_trace_is_async_signal_safe();

1156 +

1157 + if(can_safely_capture_stack_trace && !session_status.fatal.stack_trace[0]) {

1158 buffer_flush(static_save_buffer);

1159 capture_stack_trace(static_save_buffer);

1160 strncpyz(session_status.fatal.stack_trace, buffer_tostring(static_save_buffer), sizeof(session_status.fatal.stack_trace) - 1);

src/daemon/daemon.c

@@ -420,6 +420,7 @@ int become_daemon(int dont_fork, const char *user) {

420 // the child

421 gettid_uncached();

422 nd_initialize_signals();

423 + capture_stack_trace_flush();

424

425 // become session leader

426 if (setsid() < 0) {

@@ -442,6 +443,7 @@ int become_daemon(int dont_fork, const char *user) {

443 // the child

444 gettid_uncached();

445 nd_initialize_signals();

446 + capture_stack_trace_flush();

447 }

448

449 // generate our pid file

src/daemon/signal-handler.c

+68 -63

@@ -43,11 +43,16 @@ static void signal_handler(int signo) {

43 daemon_status_file_deadly_signal_received(signals_waiting[i].reason);

45 // log it

46 - char buffer[200 + 1];

47 - snprintfz(buffer, sizeof(buffer) - 1, "\nSIGNAL HANDLER: received: %s in thread %d!\n",

48 - signals_waiting[i].name, gettid_cached());

49 -

50 - if(write(STDERR_FILENO, buffer, strlen(buffer)) == -1) {

46 + char b[512];

47 + strncpyz(b, "SIGNAL HANDLER: received deadly signal: ", sizeof(b) - 1);

48 + strcat(b, signals_waiting[i].name);

49 + strcat(b, " in thread ");

50 + print_uint64(&b[strlen(b)], gettid_cached());

51 + strcat(b, " ");

52 + strcat(b, nd_thread_tag_async_safe());

53 + strcat(b, "!\n");

54 +

55 + if(write(STDERR_FILENO, b, strlen(b)) == -1) {

56 // nothing to do - we cannot write but there is no way to complain about it

57 ;

58 }

@@ -106,68 +111,68 @@ void nd_initialize_signals(void) {

111 }

112 }

113

114 +static void process_triggered_signals(void) {

115 + size_t found;

116 + do {

117 + found = 0;

118 + for (size_t i = 0; i < _countof(signals_waiting) ; i++) {

119 + if (!signals_waiting[i].count)

120 + continue;

121 +

122 + found++;

123 + signals_waiting[i].count = 0;

124 + const char *name = signals_waiting[i].name;

125 +

126 + switch (signals_waiting[i].action) {

127 + case NETDATA_SIGNAL_RELOAD_HEALTH:

128 + nd_log_limits_unlimited();

129 + netdata_log_info("SIGNAL: Received %s. Reloading HEALTH configuration...", name);

130 + nd_log_limits_reset();

131 + execute_command(CMD_RELOAD_HEALTH, NULL, NULL);

132 + break;

133 +

134 + case NETDATA_SIGNAL_REOPEN_LOGS:

135 + nd_log_limits_unlimited();

136 + netdata_log_info("SIGNAL: Received %s. Reopening all log files...", name);

137 + nd_log_limits_reset();

138 + execute_command(CMD_REOPEN_LOGS, NULL, NULL);

139 + break;

140 +

141 + case NETDATA_SIGNAL_EXIT_CLEANLY:

142 + nd_log_limits_unlimited();

143 + netdata_log_info("SIGNAL: Received %s. Cleaning up to exit...", name);

144 + commands_exit();

145 + netdata_cleanup_and_exit(signals_waiting[i].reason, NULL, NULL, NULL);

146 + exit(0);

147 + break;

148 +

149 + case NETDATA_SIGNAL_DEADLY:

150 + nd_log_limits_unlimited();

151 + daemon_status_file_deadly_signal_received(signals_waiting[i].reason);

152 + _exit(1);

153 + break;

154 +

155 + default:

156 + netdata_log_info("SIGNAL: Received %s. No signal handler configured. Ignoring it.", name);

157 + break;

158 + }

159 + }

160 + } while(found);

161 +}

162 +

163 void nd_process_signals(void) {

164 posix_unmask_my_signals();

165 + const usec_t save_every_ut = 15 * 60 * USEC_PER_SEC;

166 + usec_t last_update_mt = now_monotonic_usec();

167

112 - while(true) {

113 - // pause() causes the calling process (or thread) to sleep until a signal

114 - // is delivered that either terminates the process or causes the invocation

115 - // of a signal-catching function.

116 - if(pause() == -1 && errno == EINTR) {

168 + while (true) {

169 + usec_t mt = now_monotonic_usec();

170 + if ((mt - last_update_mt) >= save_every_ut) {

171 daemon_status_file_update_status(DAEMON_STATUS_NONE);

118 - errno_clear();

119 -

120 - // loop once, but keep looping while signals are coming in,

121 - // this is needed because a few operations may take some time

122 - // so we need to check for new signals before pausing again

123 - size_t found;

124 - do {

125 - found = 0;

126 - for (size_t i = 0; i < _countof(signals_waiting) ; i++) {

127 - if (!signals_waiting[i].count)

128 - continue;

129 -

130 - found++;

131 - signals_waiting[i].count = 0;

132 - const char *name = signals_waiting[i].name;

133 -

134 - switch (signals_waiting[i].action) {

135 - case NETDATA_SIGNAL_RELOAD_HEALTH:

136 - nd_log_limits_unlimited();

137 - netdata_log_info("SIGNAL: Received %s. Reloading HEALTH configuration...", name);

138 - nd_log_limits_reset();

139 - execute_command(CMD_RELOAD_HEALTH, NULL, NULL);

140 - break;

141 -

142 - case NETDATA_SIGNAL_REOPEN_LOGS:

143 - nd_log_limits_unlimited();

144 - netdata_log_info("SIGNAL: Received %s. Reopening all log files...", name);

145 - nd_log_limits_reset();

146 - execute_command(CMD_REOPEN_LOGS, NULL, NULL);

147 - break;

148 -

149 - case NETDATA_SIGNAL_EXIT_CLEANLY:

150 - nd_log_limits_unlimited();

151 - netdata_log_info("SIGNAL: Received %s. Cleaning up to exit...", name);

152 - commands_exit();

153 - netdata_cleanup_and_exit(signals_waiting[i].reason, NULL, NULL, NULL);

154 - exit(0);

155 - break;

156 -

157 - case NETDATA_SIGNAL_DEADLY:

158 - nd_log_limits_unlimited();

159 - daemon_status_file_deadly_signal_received(signals_waiting[i].reason);

160 - _exit(1);

161 - break;

162 -

163 - default:

164 - netdata_log_info("SIGNAL: Received %s. No signal handler configured. Ignoring it.", name);

165 - break;

166 - }

167 - }

168 - } while(found);

172 + last_update_mt += save_every_ut;

173 }

170 - else

171 - netdata_log_error("SIGNAL: pause() returned but it was not interrupted by a signal.");

174 +

175 + poll(NULL, 0, 13 * MSEC_PER_SEC + 379);

176 + process_triggered_signals();

177 }

178 }

src/libnetdata/log/nd_log-init.c

@@ -135,6 +135,8 @@ void nd_log_initialize_for_external_plugins(const char *name) {

135 nd_log.sources[NDLS_COLLECTORS].fp = NULL;

136

137 // nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));

138 +

139 + capture_stack_trace_init();

140 }

141

142 // --------------------------------------------------------------------------------------------------------------------

@@ -263,6 +265,8 @@ void nd_log_initialize(void) {

265

266 for(size_t i = 0 ; i < _NDLS_MAX ; i++)

267 nd_log_open(&nd_log.sources[i], i);

268 +

269 + capture_stack_trace_init();

270 }

271

272 void nd_log_reopen_log_files(bool log) {

@@ -285,6 +289,7 @@ void nd_log_reopen_log_files_for_spawn_server(const char *name) {

289 nd_log.fatal_final_cb = NULL;

290

291 gettid_uncached();

292 + capture_stack_trace_flush();

293

294 if(nd_log.syslog.initialized) {

295 closelog();

src/libnetdata/log/nd_log-stacktrace.c

+82 -8

@@ -7,9 +7,24 @@ bool nd_log_forked = false;

7 #define NO_STACK_TRACE_PREFIX "stack trace not available: "

9 #if defined(HAVE_LIBUNWIND)

10 +#define UNW_LOCAL_ONLY

11 #include <libunwind.h>

13 +void capture_stack_trace_init(void) {

14 + unw_set_caching_policy(unw_local_addr_space, UNW_CACHE_NONE);

15 +}

16 +

17 +void capture_stack_trace_flush(void) {

18 + unw_flush_cache(unw_local_addr_space, 0, 0);

19 +}

20 +

21 +bool capture_stack_trace_is_async_signal_safe(void) {

22 + return true;

23 +}

24 +

25 void capture_stack_trace(BUFFER *wb) {

26 + // this function is async-signal-safe, if the buffer has enough space to hold the stack trace

27 +

28 unw_cursor_t cursor;

29 unw_context_t context;

30 size_t frames = 0;

@@ -28,14 +43,22 @@ void capture_stack_trace(BUFFER *wb) {

43 break;

45 const char *name = sym;

31 - if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) == 0) {

32 - if (frames++) buffer_strcat(wb, "\n");

33 - buffer_sprintf(wb, "#%d %s+0x%lx", added, name, (unsigned long)offset);

46 + if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) != 0) {

47 + name = "<unknown>";

48 + offset = 0;

49 }

35 - else {

36 - if (frames++)

37 - buffer_strcat(wb, "\n");

38 - buffer_sprintf(wb, "#%d <unknown>", added);

50 +

51 + if (frames++)

52 + buffer_putc(wb, '\n');

53 +

54 + buffer_putc(wb, '#');

55 + buffer_print_uint64(wb, added);

56 + buffer_putc(wb, ' ');

57 + buffer_strcat(wb, name);

58 +

59 + if(offset) {

60 + buffer_putc(wb, '+');

61 + buffer_print_uint64_hex(wb, offset);

62 }

64 added++;

@@ -47,6 +70,18 @@ void capture_stack_trace(BUFFER *wb) {

71 #elif defined(HAVE_BACKTRACE)

73 +void capture_stack_trace_init(void) {

74 + ;

75 +}

76 +

77 +void capture_stack_trace_flush(void) {

78 + ;

79 +}

80 +

81 +bool capture_stack_trace_is_async_signal_safe(void) {

82 + return false;

83 +}

84 +

85 void capture_stack_trace(BUFFER *wb) {

86 void *array[50];

87 char **messages;

@@ -64,9 +99,32 @@ void capture_stack_trace(BUFFER *wb) {

99 // Format the stack trace (removing the address part)

100 for (i = 0; i < size; i++) {

101 if(messages[i] && *messages[i]) {

102 + if(added)

103 + buffer_putc(wb, '\n');

104 +

105 +#if defined(OS_MACOS)

106 + // remove the address part

107 + char *p = strstr(messages[i], "0x");

108 + char *e = p ? strchr(p, ' ') : NULL;

109 + if (e) {

110 + e++;

111 + buffer_putc(wb, '#');

112 + buffer_print_uint64(wb, added);

113 + buffer_putc(wb, ' ');

114 + buffer_strcat(wb, e);

115 + }

116 + else

117 + buffer_strcat(wb, messages[i]);

118 +#else

119 + buffer_putc(wb, '#');

120 + buffer_print_uint64(wb, i);

121 + buffer_putc(wb, ' ');

122 +

123 + // remove the address part

124 char *p = strstr(messages[i], " [");

125 size_t len = p ? (size_t)(p - messages[i]) : strlen(messages[i]);

69 - buffer_sprintf(wb, "#%d %.*s\n", i, (int)len, messages[i]);

126 + buffer_fast_strcat(wb, messages[i], len);

127 +#endif

128 added++;

129 }

130 }

@@ -79,8 +137,24 @@ void capture_stack_trace(BUFFER *wb) {

137

138 #else

139

140 +void capture_stack_trace_init(void) {

141 + ;

142 +}

143 +

144 +void capture_stack_trace_flush(void) {

145 + ;

146 +}

147 +

148 +bool capture_stack_trace_is_async_signal_safe(void) {

149 + return false;

150 +}

151 +

152 void capture_stack_trace(BUFFER *wb) {

153 buffer_strcat(wb, NO_STACK_TRACE_PREFIX "no back-end available");

154 +

155 + // probably we can have something like this?

156 + // https://maskray.me/blog/2022-04-09-unwinding-through-signal-handler

157 + // (at the end - but it needs the frame pointer)

158 }

159

160 #endif

src/libnetdata/log/nd_log.h

@@ -32,7 +32,11 @@ int nd_log_priority2id(const char *priority);

32 const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);

33 const char *nd_log_method_for_external_plugins(const char *s);

34 ND_UUID nd_log_get_invocation_id(void);

35 +

36 void capture_stack_trace(BUFFER *wb);

37 +void capture_stack_trace_init(void);

38 +void capture_stack_trace_flush(void);

39 +bool capture_stack_trace_is_async_signal_safe(void);

41 typedef void (*log_event_t)(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);

42 void nd_log_register_fatal_data_cb(log_event_t cb);

src/libnetdata/threads/threads.c

@@ -130,6 +130,13 @@ const char *nd_thread_tag(void) {

130 return nd_thread_get_name(false);

131 }

132

133 +const char *nd_thread_tag_async_safe(void) {

134 + if(nd_thread_has_tag())

135 + return _nd_thread_info->tag;

136 +

137 + return _nd_thread_os_name;

138 +}

139 +

140 void nd_thread_tag_set(const char *tag) {

141 if(!tag || !*tag) return;

142

src/libnetdata/threads/threads.h

@@ -63,6 +63,7 @@ struct netdata_static_thread {

64 #define NETDATA_THREAD_TAG_MAX 100

65 const char *nd_thread_tag(void);

66 +const char *nd_thread_tag_async_safe(void);

67 int nd_thread_has_tag(void);

69 #define THREAD_TAG_STREAM_RECEIVER "RCVR"