detect when running in CI and disable posting status (#19787)
* detect when running in CI and disable posting status * unify stack trace reporting * simplify signal handler
Costa Tsaousis committed
Mar 6, 2025 at 15:52 UTC
fb67eb9c9cf3cf0f3a657f41071115b627dec4cf
3 files changed
+111
-105
src/daemon/daemon-status-file.c
+14
-2
@@ -9,7 +9,7 @@
9
#include <openssl/pem.h>
10
#include <openssl/err.h>
11
12
-#define STATUS_FILE_VERSION 10
12
+#define STATUS_FILE_VERSION 11
13
14
#define STATUS_FILENAME "status-netdata.json"
15
@@ -558,6 +558,13 @@ void daemon_status_file_load(DAEMON_STATUS_FILE *ds) {
558
static bool save_status_file(const char *directory, const char *content, size_t content_size) {
559
// THIS FUNCTION MUST USE ONLY ASYNC-SAFE OPERATIONS
560
561
+ // Linux: https://man7.org/linux/man-pages/man7/signal-safety.7.html
562
+ // memcpy(), strlen(), open(), write(), fsync(), close(), chmod(), rename(), unlink()
563
+
564
+ // MacOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/sigaction.2.html#//apple_ref/doc/man/2/sigaction
565
+ // open(), write(), fsync(), close(), chmod(), rename(), unlink()
566
+ // does not explicitly mention memcpy() and strlen(), but they are safe
567
+
568
if(!directory || !*directory)
569
return false;
570
@@ -810,6 +817,11 @@ struct log_priority PRI_USER_SHOULD_FIX = { NDLP_WARNING, NDLP_INFO };
817
struct log_priority PRI_NETDATA_BUG = { NDLP_CRIT, NDLP_ERR };
818
struct log_priority PRI_BAD_BUT_NO_REASON = { NDLP_ERR, NDLP_WARNING };
819
820
+static bool is_ci(void) {
821
+ const char *ci = getenv("CI");
822
+ return ci && *ci && strcasecmp(ci, "true") == 0;
823
+}
824
+
825
void daemon_status_file_check_crash(void) {
826
FUNCTION_RUN_ONCE();
827
@@ -1015,7 +1027,7 @@ void daemon_status_file_check_crash(void) {
1027
1028
// check if we have already posted this crash in the last 24 hours
1029
XXH64_hash_t hash = daemon_status_file_hash(&last_session_status, msg, cause);
1018
- if(dedup_already_posted(&session_status, hash))
1030
+ if(dedup_already_posted(&session_status, hash) || (last_session_status.restarts < 10 && is_ci()))
1031
disable_crash_report = true;
1032
1033
if(!disable_crash_report && (analytics_check_enabled() || post_crash_report)) {
src/daemon/signal-handler.c
+71
-86
@@ -4,7 +4,6 @@
4
#include "daemon/daemon-status-file.h"
5
6
typedef enum signal_action {
7
- NETDATA_SIGNAL_END_OF_LIST,
7
NETDATA_SIGNAL_IGNORE,
8
NETDATA_SIGNAL_EXIT_CLEANLY,
9
NETDATA_SIGNAL_REOPEN_LOGS,
@@ -30,54 +29,43 @@ static struct {
29
{ SIGFPE, "SIGFPE", 0, NETDATA_SIGNAL_DEADLY, EXIT_REASON_SIGFPE },
30
{ SIGILL, "SIGILL", 0, NETDATA_SIGNAL_DEADLY, EXIT_REASON_SIGILL },
31
{ SIGABRT, "SIGABRT", 0, NETDATA_SIGNAL_DEADLY, EXIT_REASON_SIGABRT },
33
-
34
- // terminator
35
- { 0, "NONE", 0, NETDATA_SIGNAL_END_OF_LIST, 0 }
32
};
33
34
static void signal_handler(int signo) {
39
- static size_t recurse = 0;
40
- if(__atomic_add_fetch(&recurse, 1, __ATOMIC_RELAXED) > 1) {
41
- __atomic_sub_fetch(&recurse, 1, __ATOMIC_RELAXED);
42
- return;
43
- }
44
-
45
- int i;
46
- for(i = 0; signals_waiting[i].action != NETDATA_SIGNAL_END_OF_LIST; i++) {
47
- if(unlikely(signals_waiting[i].signo == signo)) {
48
- signals_waiting[i].count++;
35
+ for(size_t i = 0; i < _countof(signals_waiting) ; i++) {
36
+ if(signals_waiting[i].signo != signo)
37
+ continue;
38
50
- if(signals_waiting[i].action == NETDATA_SIGNAL_DEADLY) {
51
- // Update the status file
52
- daemon_status_file_deadly_signal_received(signals_waiting[i].reason);
39
+ signals_waiting[i].count++;
40
54
- // log it
55
- char buffer[200 + 1];
56
- snprintfz(buffer, sizeof(buffer) - 1, "\nSIGNAL HANDLER: received: %s in thread %d!\n",
57
- signals_waiting[i].name, gettid_cached());
41
+ if(signals_waiting[i].action == NETDATA_SIGNAL_DEADLY) {
42
+ // Update the status file
43
+ daemon_status_file_deadly_signal_received(signals_waiting[i].reason);
44
59
- if(write(STDERR_FILENO, buffer, strlen(buffer)) == -1) {
60
- // nothing to do - we cannot write but there is no way to complain about it
61
- ;
62
- }
45
+ // log it
46
+ char buffer[200 + 1];
47
+ snprintfz(buffer, sizeof(buffer) - 1, "\nSIGNAL HANDLER: received: %s in thread %d!\n",
48
+ signals_waiting[i].name, gettid_cached());
49
64
- // Reset the signal's disposition to the default handler.
50
+ if(write(STDERR_FILENO, buffer, strlen(buffer)) == -1) {
51
+ // nothing to do - we cannot write but there is no way to complain about it
52
+ ;
53
+ }
54
66
- struct sigaction sa;
67
- sa.sa_handler = SIG_DFL;
68
- sigemptyset(&sa.sa_mask);
69
- sa.sa_flags = 0;
70
- sigaction(signo, &sa, NULL);
55
+ // Reset the signal's disposition to the default handler.
56
72
- // Re-raise the signal, which now uses the default action.
73
- raise(signo);
74
- }
57
+ struct sigaction sa;
58
+ sa.sa_handler = SIG_DFL;
59
+ sigemptyset(&sa.sa_mask);
60
+ sa.sa_flags = 0;
61
+ sigaction(signo, &sa, NULL);
62
76
- break;
63
+ // Re-raise the signal, which now uses the default action.
64
+ raise(signo);
65
}
78
- }
66
80
- __atomic_sub_fetch(&recurse, 1, __ATOMIC_RELAXED);
67
+ break;
68
+ }
69
}
70
71
// Unmask all signals the netdata main signal handler uses.
@@ -86,7 +74,7 @@ static void posix_unmask_my_signals(void) {
74
sigset_t sigset;
75
sigemptyset(&sigset);
76
89
- for (int i = 0; signals_waiting[i].action != NETDATA_SIGNAL_END_OF_LIST; i++)
77
+ for (size_t i = 0; i < _countof(signals_waiting) ; i++)
78
sigaddset(&sigset, signals_waiting[i].signo);
79
80
if (pthread_sigmask(SIG_UNBLOCK, &sigset, NULL) != 0)
@@ -103,8 +91,7 @@ void nd_initialize_signals(void) {
91
// ignore all signals while we run in a signal handler
92
sigfillset(&sa.sa_mask);
93
106
- int i;
107
- for (i = 0; signals_waiting[i].action != NETDATA_SIGNAL_END_OF_LIST; i++) {
94
+ for (size_t i = 0; i < _countof(signals_waiting) ; i++) {
95
switch (signals_waiting[i].action) {
96
case NETDATA_SIGNAL_IGNORE:
97
sa.sa_handler = SIG_IGN;
@@ -122,7 +109,7 @@ void nd_initialize_signals(void) {
109
void nd_process_signals(void) {
110
posix_unmask_my_signals();
111
125
- while(1) {
112
+ while(true) {
113
// pause() causes the calling process (or thread) to sleep until a signal
114
// is delivered that either terminates the process or causes the invocation
115
// of a signal-catching function.
@@ -133,54 +120,52 @@ void nd_process_signals(void) {
120
// loop once, but keep looping while signals are coming in,
121
// this is needed because a few operations may take some time
122
// so we need to check for new signals before pausing again
136
- int found = 1;
137
- while(found) {
123
+ size_t found;
124
+ do {
125
found = 0;
139
-
140
- // execute the actions of the signals
141
- int i;
142
- for (i = 0; signals_waiting[i].action != NETDATA_SIGNAL_END_OF_LIST; i++) {
143
- if (signals_waiting[i].count) {
144
- found = 1;
145
- signals_waiting[i].count = 0;
146
- const char *name = signals_waiting[i].name;
147
-
148
- switch (signals_waiting[i].action) {
149
- case NETDATA_SIGNAL_RELOAD_HEALTH:
150
- nd_log_limits_unlimited();
151
- netdata_log_info("SIGNAL: Received %s. Reloading HEALTH configuration...", name);
152
- nd_log_limits_reset();
153
- execute_command(CMD_RELOAD_HEALTH, NULL, NULL);
154
- break;
155
-
156
- case NETDATA_SIGNAL_REOPEN_LOGS:
157
- nd_log_limits_unlimited();
158
- netdata_log_info("SIGNAL: Received %s. Reopening all log files...", name);
159
- nd_log_limits_reset();
160
- execute_command(CMD_REOPEN_LOGS, NULL, NULL);
161
- break;
162
-
163
- case NETDATA_SIGNAL_EXIT_CLEANLY:
164
- nd_log_limits_unlimited();
165
- netdata_log_info("SIGNAL: Received %s. Cleaning up to exit...", name);
166
- commands_exit();
167
- netdata_cleanup_and_exit(signals_waiting[i].reason, NULL, NULL, NULL);
168
- exit(0);
169
- break;
170
-
171
- case NETDATA_SIGNAL_DEADLY:
172
- nd_log_limits_unlimited();
173
- daemon_status_file_deadly_signal_received(signals_waiting[i].reason);
174
- _exit(1);
175
- break;
176
-
177
- default:
178
- netdata_log_info("SIGNAL: Received %s. No signal handler configured. Ignoring it.", name);
179
- break;
180
- }
126
+ for (size_t i = 0; i < _countof(signals_waiting) ; i++) {
127
+ if (!signals_waiting[i].count)
128
+ continue;
129
+
130
+ found++;
131
+ signals_waiting[i].count = 0;
132
+ const char *name = signals_waiting[i].name;
133
+
134
+ switch (signals_waiting[i].action) {
135
+ case NETDATA_SIGNAL_RELOAD_HEALTH:
136
+ nd_log_limits_unlimited();
137
+ netdata_log_info("SIGNAL: Received %s. Reloading HEALTH configuration...", name);
138
+ nd_log_limits_reset();
139
+ execute_command(CMD_RELOAD_HEALTH, NULL, NULL);
140
+ break;
141
+
142
+ case NETDATA_SIGNAL_REOPEN_LOGS:
143
+ nd_log_limits_unlimited();
144
+ netdata_log_info("SIGNAL: Received %s. Reopening all log files...", name);
145
+ nd_log_limits_reset();
146
+ execute_command(CMD_REOPEN_LOGS, NULL, NULL);
147
+ break;
148
+
149
+ case NETDATA_SIGNAL_EXIT_CLEANLY:
150
+ nd_log_limits_unlimited();
151
+ netdata_log_info("SIGNAL: Received %s. Cleaning up to exit...", name);
152
+ commands_exit();
153
+ netdata_cleanup_and_exit(signals_waiting[i].reason, NULL, NULL, NULL);
154
+ exit(0);
155
+ break;
156
+
157
+ case NETDATA_SIGNAL_DEADLY:
158
+ nd_log_limits_unlimited();
159
+ daemon_status_file_deadly_signal_received(signals_waiting[i].reason);
160
+ _exit(1);
161
+ break;
162
+
163
+ default:
164
+ netdata_log_info("SIGNAL: Received %s. No signal handler configured. Ignoring it.", name);
165
+ break;
166
}
167
}
183
- }
168
+ } while(found);
169
}
170
else
171
netdata_log_error("SIGNAL: pause() returned but it was not interrupted by a signal.");
src/libnetdata/log/nd_log-stacktrace.c
+26
-17
@@ -4,6 +4,8 @@
4
5
bool nd_log_forked = false;
6
7
+#define NO_STACK_TRACE_PREFIX "stack trace not available: "
8
+
9
#if defined(HAVE_LIBUNWIND)
10
#include <libunwind.h>
11
@@ -16,31 +18,31 @@ void capture_stack_trace(BUFFER *wb) {
18
unw_getcontext(&context);
19
unw_init_local(&cursor, &context);
20
19
- // // Skip first 3 frames (our logging infrastructure)
20
- // for (int i = 0; i < 3; i++) {
21
- // if (unw_step(&cursor) <= 0)
22
- // goto cleanup; // Ensure proper cleanup if unwinding fails early
23
- // }
24
-
21
+ size_t added = 0;
22
while (unw_step(&cursor) > 0) {
23
unw_word_t offset, pc;
24
char sym[256];
25
26
unw_get_reg(&cursor, UNW_REG_IP, &pc);
30
- if (pc == 0)
27
+ if (!pc)
28
break;
29
30
const char *name = sym;
31
if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) == 0) {
32
if (frames++) buffer_strcat(wb, "\n");
36
- buffer_sprintf(wb, "%s+0x%lx", name, (unsigned long)offset);
33
+ buffer_sprintf(wb, "#%d %s+0x%lx", added, name, (unsigned long)offset);
34
}
35
else {
36
if (frames++)
37
buffer_strcat(wb, "\n");
41
- buffer_strcat(wb, "<unknown>");
38
+ buffer_sprintf(wb, "#%d <unknown>", added);
39
}
40
+
41
+ added++;
42
}
43
+
44
+ if (!added)
45
+ buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libunwind reports no frames");
46
}
47
48
#elif defined(HAVE_BACKTRACE)
@@ -53,25 +55,32 @@ void capture_stack_trace(BUFFER *wb) {
55
size = backtrace(array, _countof(array));
56
messages = backtrace_symbols(array, size);
57
56
- if (messages == NULL) {
57
- buffer_strcat(wb, "backtrace failed to get symbols");
58
+ if (!messages) {
59
+ buffer_strcat(wb, NO_STACK_TRACE_PREFIX "backtrace() reports no symbols");
60
return;
61
}
62
63
+ size_t added = 0;
64
// Format the stack trace (removing the address part)
65
for (i = 0; i < size; i++) {
63
- char *p = strstr(messages[i], " [");
64
- size_t len = p ? (size_t)(p - messages[i]) : strlen(messages[i]);
65
- buffer_sprintf(wb, "#%d %.*s\n", i, (int)len, messages[i]);
66
+ if(messages[i] && *messages[i]) {
67
+ char *p = strstr(messages[i], " [");
68
+ size_t len = p ? (size_t)(p - messages[i]) : strlen(messages[i]);
69
+ buffer_sprintf(wb, "#%d %.*s\n", i, (int)len, messages[i]);
70
+ added++;
71
+ }
72
}
73
74
+ if(!added)
75
+ buffer_strcat(wb, NO_STACK_TRACE_PREFIX "backtrace() reports no frames");
76
+
77
free(messages);
78
}
79
80
#else
81
82
void capture_stack_trace(BUFFER *wb) {
74
- buffer_strcat(wb, "stack trace not available");
83
+ buffer_strcat(wb, NO_STACK_TRACE_PREFIX "no back-end available");
84
}
85
86
#endif
@@ -81,13 +90,13 @@ bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {
90
91
if (nd_log_forked) {
92
// libunwind freezes in forked children
84
- buffer_strcat(wb, "stack trace after fork is disabled");
93
+ buffer_strcat(wb, NO_STACK_TRACE_PREFIX "stack trace after fork is disabled");
94
return true;
95
}
96
97
if (in_stack_trace) {
98
// Prevent recursion
90
- buffer_strcat(wb, "stack trace recursion detected");
99
+ buffer_strcat(wb, NO_STACK_TRACE_PREFIX "stack trace recursion detected");
100
return true;
101
}
102