sentry events annotations (#19872)
* have a separate entry for sentry on deadly events * always save fatal info into the file, overwriting what was already there * added fatal annotations in sentry reports * added fatal thread * enable also the fingerprint
Costa Tsaousis committed
Mar 15, 2025 at 20:35 UTC
8850b577b0eb0276d63c3d0b04a9c48b22516782
8 files changed
+202
-43
src/daemon/daemon-shutdown.c
+2
-4
@@ -374,11 +374,9 @@ void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char
374
375
#ifdef ENABLE_SENTRY
376
if (ret && abort_on_fatal) {
377
- if (action_data) {
378
- nd_sentry_add_breadcrumb(action_data);
379
- }
377
abort();
381
- } else {
378
+ }
379
+ else {
380
nd_sentry_fini();
381
curl_global_cleanup();
382
exit(ret);
src/daemon/daemon-status-file.c
+96
-18
@@ -9,7 +9,9 @@
9
#include <openssl/pem.h>
10
#include <openssl/err.h>
11
12
-#define STATUS_FILE_VERSION 17
12
+#ifdef ENABLE_SENTRY
13
+#include "sentry-native/sentry-native.h"
14
+#endif
15
16
#define STATUS_FILENAME "status-netdata.json"
17
@@ -98,7 +100,8 @@ static uint64_t daemon_status_file_hash(DAEMON_STATUS_FILE *ds, const char *msg,
100
RRD_DB_MODE db_mode;
101
uint8_t db_tiers;
102
bool kubernetes;
101
- bool sentry;
103
+ bool sentry_available;
104
+ bool sentry_fatal;
105
ND_UUID host_id;
106
ND_UUID machine_id;
107
long line;
@@ -119,7 +122,8 @@ static uint64_t daemon_status_file_hash(DAEMON_STATUS_FILE *ds, const char *msg,
122
.db_mode = ds->db_mode,
123
.db_tiers = ds->db_tiers,
124
.kubernetes = ds->kubernetes,
122
- .sentry = ds->sentry,
125
+ .sentry_available = ds->sentry_available,
126
+ .sentry_fatal = ds->fatal.sentry,
127
.host_id = ds->host_id,
128
.machine_id = ds->machine_id,
129
};
@@ -172,9 +176,8 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
176
buffer_json_member_add_boolean(wb, "ND_kubernetes", ds->kubernetes); // custom
177
}
178
175
- if(ds->v >= 16) {
176
- buffer_json_member_add_boolean(wb, "ND_sentry", ds->sentry); // custom
177
- }
179
+ if(ds->v >= 16)
180
+ buffer_json_member_add_boolean(wb, "ND_sentry_available", ds->sentry_available); // custom
181
182
buffer_json_member_add_object(wb, "ND_timings"); // custom
183
{
@@ -248,6 +251,9 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
251
SIGNAL_CODE_2str_h(ds->fatal.signal_code, signal_code, sizeof(signal_code));
252
buffer_json_member_add_string_or_empty(wb, "signal_code", signal_code);
253
}
254
+
255
+ if(ds->v >= 17)
256
+ buffer_json_member_add_boolean(wb, "sentry", ds->fatal.sentry);
257
}
258
buffer_json_object_close(wb);
259
@@ -332,9 +338,10 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *
338
ds->kubernetes = false;
339
}
340
335
- if(version >= 16) {
336
- JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, "ND_sentry", ds->sentry, error, required_v16);
337
- }
341
+ if(version >= 17)
342
+ JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, "ND_sentry_available", ds->sentry_available, error, required_v17);
343
+ else if(version == 16)
344
+ JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, "ND_sentry", ds->sentry_available, error, required_v16);
345
});
346
347
// Parse host object
@@ -392,6 +399,9 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *
399
if(version >= 16) {
400
JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "signal_code", SIGNAL_CODE_2id_h, ds->fatal.signal_code, error, required_v16);
401
}
402
+ if(version >= 17) {
403
+ JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "sentry", SIGNAL_CODE_2id_h, ds->fatal.sentry, error, required_v17);
404
+ }
405
});
406
407
// Parse the last posted object
@@ -474,9 +484,9 @@ static void daemon_status_file_refresh(DAEMON_STATUS status) {
484
session_status.db_tiers = nd_profile.storage_tiers;
485
486
#if defined(ENABLE_SENTRY)
477
- session_status.sentry = true;
487
+ session_status.sentry_available = true;
488
#else
479
- session_status.sentry = false;
489
+ session_status.sentry_available = false;
490
#endif
491
492
session_status.claim_id = claim_id_get_uuid();
@@ -1222,22 +1232,22 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio
1232
1233
copy_and_clean_thread_name_if_empty(&session_status, nd_thread_tag());
1234
1225
- if(!session_status.fatal.filename[0] && filename)
1235
+ if(filename && *filename)
1236
strncpyz(session_status.fatal.filename, filename, sizeof(session_status.fatal.filename) - 1);
1237
1228
- if(!session_status.fatal.function[0] && function)
1238
+ if(function && *function)
1239
strncpyz(session_status.fatal.function, function, sizeof(session_status.fatal.function) - 1);
1240
1231
- if(!session_status.fatal.message[0] && message)
1241
+ if(message && *message)
1242
strncpyz(session_status.fatal.message, message, sizeof(session_status.fatal.message) - 1);
1243
1234
- if(!session_status.fatal.errno_str[0] && errno_str)
1244
+ if(errno_str && *errno_str)
1245
strncpyz(session_status.fatal.errno_str, errno_str, sizeof(session_status.fatal.errno_str) - 1);
1246
1237
- if(stack_trace_is_empty(&session_status) && stack_trace)
1247
+ if(stack_trace && *stack_trace && stack_trace_is_empty(&session_status))
1248
strncpyz(session_status.fatal.stack_trace, stack_trace, sizeof(session_status.fatal.stack_trace) - 1);
1249
1240
- if(!session_status.fatal.line)
1250
+ if(line)
1251
session_status.fatal.line = line;
1252
1253
spinlock_unlock(&session_status.fatal.spinlock);
@@ -1251,6 +1261,10 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio
1261
freez((void *)message);
1262
freez((void *)errno_str);
1263
freez((void *)stack_trace);
1264
+
1265
+#ifdef ENABLE_SENTRY
1266
+ nd_sentry_add_fatal_message_as_breadcrumb();
1267
+#endif
1268
}
1269
1270
// --------------------------------------------------------------------------------------------------------------------
@@ -1287,7 +1301,7 @@ bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE c
1301
dsf_acquire(session_status);
1302
1303
session_status.exit_reason |= reason;
1290
- session_status.sentry = chained_handler;
1304
+ session_status.fatal.sentry = chained_handler;
1305
1306
if(code)
1307
session_status.fatal.signal_code = code;
@@ -1364,3 +1378,67 @@ void daemon_status_file_shutdown_step(const char *step) {
1378
1379
daemon_status_file_update_status(DAEMON_STATUS_EXITING);
1380
}
1381
+
1382
+// --------------------------------------------------------------------------------------------------------------------
1383
+// public API to get values
1384
+
1385
+const char *daemon_status_file_get_install_type(void) {
1386
+ return session_status.install_type;
1387
+}
1388
+
1389
+const char *daemon_status_file_get_architecture(void) {
1390
+ return session_status.architecture;
1391
+}
1392
+
1393
+const char *daemon_status_file_get_virtualization(void) {
1394
+ return session_status.virtualization;
1395
+}
1396
+
1397
+const char *daemon_status_file_get_container(void) {
1398
+ return session_status.container;
1399
+}
1400
+
1401
+const char *daemon_status_file_get_os_name(void) {
1402
+ return session_status.os_name;
1403
+}
1404
+
1405
+const char *daemon_status_file_get_os_version(void) {
1406
+ return session_status.os_version;
1407
+}
1408
+
1409
+const char *daemon_status_file_get_os_id(void) {
1410
+ return session_status.os_id;
1411
+}
1412
+
1413
+const char *daemon_status_file_get_os_id_like(void) {
1414
+ return session_status.os_id_like;
1415
+}
1416
+
1417
+const char *daemon_status_file_get_fatal_filename(void) {
1418
+ return session_status.fatal.filename;
1419
+}
1420
+
1421
+const char *daemon_status_file_get_fatal_function(void) {
1422
+ return session_status.fatal.function;
1423
+}
1424
+
1425
+const char *daemon_status_file_get_fatal_message(void) {
1426
+ return session_status.fatal.message;
1427
+}
1428
+
1429
+const char *daemon_status_file_get_fatal_errno(void) {
1430
+ return session_status.fatal.errno_str;
1431
+}
1432
+
1433
+const char *daemon_status_file_get_fatal_stack_trace(void) {
1434
+ return session_status.fatal.stack_trace;
1435
+}
1436
+
1437
+const char *daemon_status_file_get_fatal_thread(void) {
1438
+ return session_status.fatal.thread;
1439
+}
1440
+
1441
+long daemon_status_file_get_fatal_line(void) {
1442
+ return session_status.fatal.line;
1443
+}
1444
+
src/daemon/daemon-status-file.h
+21
-1
@@ -7,6 +7,8 @@
7
#include "daemon/config/netdata-conf-profile.h"
8
#include "database/rrd-database-mode.h"
9
10
+#define STATUS_FILE_VERSION 17
11
+
12
typedef enum {
13
DAEMON_STATUS_NONE,
14
DAEMON_STATUS_INITIALIZING,
@@ -37,7 +39,7 @@ typedef struct daemon_status_file {
39
RRD_DB_MODE db_mode;
40
uint8_t db_tiers;
41
bool kubernetes;
40
- bool sentry;
42
+ bool sentry_available; // true when sentry support is compiled in
43
44
time_t boottime; // system boottime
45
time_t uptime; // netdata uptime
@@ -82,6 +84,7 @@ typedef struct daemon_status_file {
84
char stack_trace[2048];
85
char thread[ND_THREAD_TAG_MAX + 1];
86
SIGNAL_CODE signal_code;
87
+ bool sentry; // true when the error was also reported to sentry
88
} fatal;
89
90
struct {
@@ -111,4 +114,21 @@ void daemon_status_file_shutdown_step(const char *step);
114
void daemon_status_file_init(void);
115
void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);
116
117
+const char *daemon_status_file_get_install_type(void);
118
+const char *daemon_status_file_get_architecture(void);
119
+const char *daemon_status_file_get_virtualization(void);
120
+const char *daemon_status_file_get_container(void);
121
+const char *daemon_status_file_get_os_name(void);
122
+const char *daemon_status_file_get_os_version(void);
123
+const char *daemon_status_file_get_os_id(void);
124
+const char *daemon_status_file_get_os_id_like(void);
125
+
126
+const char *daemon_status_file_get_fatal_filename(void);
127
+const char *daemon_status_file_get_fatal_function(void);
128
+const char *daemon_status_file_get_fatal_message(void);
129
+const char *daemon_status_file_get_fatal_errno(void);
130
+const char *daemon_status_file_get_fatal_stack_trace(void);
131
+const char *daemon_status_file_get_fatal_thread(void);
132
+long daemon_status_file_get_fatal_line(void);
133
+
134
#endif //NETDATA_DAEMON_STATUS_FILE_H
src/daemon/daemon.c
+4
-2
@@ -1,6 +1,7 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include "common.h"
4
+#include "sentry-native/sentry-native.h"
5
#include <sched.h>
6
7
char *pidfile = NULL;
@@ -32,8 +33,9 @@ static void fix_directory_file_permissions(const char *dirname, uid_t uid, gid_t
33
closedir(dir);
34
}
35
35
-static void change_dir_ownership(const char *dir, uid_t uid, gid_t gid, bool recursive)
36
-{
36
+static void change_dir_ownership(const char *dir, uid_t uid, gid_t gid, bool recursive) {
37
+ if(!dir || !*dir) return;
38
+
39
if (chown(dir, uid, gid) == -1)
40
netdata_log_error("Cannot chown directory '%s' to %u:%u", dir, (unsigned int)uid, (unsigned int)gid);
41
src/daemon/main.c
-2
@@ -989,9 +989,7 @@ int netdata_main(int argc, char **argv) {
989
// ----------------------------------------------------------------------------------------------------------------
990
delta_startup_time("sentry");
991
992
- nd_cleanup_fatal_signals(); // avoid sentry looping deadly signals back to us
992
nd_sentry_init();
994
- nd_initialize_signals(true);
993
#endif
994
995
// ----------------------------------------------------------------------------------------------------------------
src/daemon/sentry-native/sentry-native.c
+73
-15
@@ -5,8 +5,14 @@
5
6
#include "sentry.h"
7
8
+static char sentry_path[FILENAME_MAX] = "";
9
+
10
bool nd_sentry_crash_report_enabled = true;
11
12
+const char *nd_sentry_path(void) {
13
+ return sentry_path;
14
+}
15
+
16
void nd_sentry_crash_report(bool enable) {
17
nd_sentry_crash_report_enabled = enable;
18
}
@@ -39,18 +45,19 @@ static sentry_value_t nd_sentry_before_send(
45
return event;
46
}
47
42
-void nd_sentry_init(void)
43
-{
48
+void nd_sentry_init(void) {
49
if (!analytics_check_enabled())
50
return;
51
52
// path where sentry should save stuff
48
- char path[FILENAME_MAX];
49
- snprintfz(path, FILENAME_MAX - 1, "%s/.sentry-native", netdata_configured_cache_dir);
53
+ snprintfz(sentry_path, FILENAME_MAX - 1, "%s/.sentry-native", netdata_configured_cache_dir);
54
+
55
+ // ----------------------------------------------------------------------------------------------------------------
56
+ // sentry options
57
58
sentry_options_t *options = sentry_options_new();
59
sentry_options_set_dsn(options, NETDATA_SENTRY_DSN);
53
- sentry_options_set_database_path(options, path);
60
+ sentry_options_set_database_path(options, sentry_path);
61
sentry_options_set_environment(options, NETDATA_SENTRY_ENVIRONMENT);
62
63
if(NETDATA_VERSION[0] == 'v')
@@ -63,42 +70,93 @@ void nd_sentry_init(void)
70
sentry_options_set_debug(options, 1);
71
#endif
72
73
+ sentry_options_set_on_crash(options, nd_sentry_on_crash, NULL);
74
+ sentry_options_set_before_send(options, nd_sentry_before_send, NULL);
75
+
76
+ // ----------------------------------------------------------------------------------------------------------------
77
+ // initialization
78
+
79
+ nd_cleanup_fatal_signals();
80
+ sentry_init(options);
81
+ nd_initialize_signals(true);
82
+
83
+ // ----------------------------------------------------------------------------------------------------------------
84
+ // tags
85
+
86
+ sentry_set_tag("install_type", daemon_status_file_get_install_type());
87
+ sentry_set_tag("architecture", daemon_status_file_get_architecture());
88
+ sentry_set_tag("virtualization", daemon_status_file_get_virtualization());
89
+ sentry_set_tag("container", daemon_status_file_get_container());
90
+ sentry_set_tag("os_name", daemon_status_file_get_os_name());
91
+ sentry_set_tag("os_version", daemon_status_file_get_os_version());
92
+ sentry_set_tag("os_id", daemon_status_file_get_os_id());
93
+ sentry_set_tag("os_id_like", daemon_status_file_get_os_id_like());
94
+
95
+ // profile
96
CLEAN_BUFFER *profile = buffer_create(0, NULL);
97
ND_PROFILE_2buffer(profile, nd_profile_detect_and_configure(false), " ");
98
sentry_set_tag("profile", buffer_tostring(profile));
99
100
+ // db_mode
101
sentry_set_tag("db_mode", rrd_memory_mode_name(default_rrd_memory_mode));
102
103
+ // db_tiers
104
char tiers[UINT64_MAX_LENGTH];
105
print_uint64(tiers, nd_profile.storage_tiers);
106
sentry_set_tag("db_tiers", tiers);
107
76
- sentry_options_set_on_crash(options, nd_sentry_on_crash, NULL);
77
- sentry_options_set_before_send(options, nd_sentry_before_send, NULL);
108
+ // invocation_id
109
+ ND_UUID invocation_id = nd_log_get_invocation_id();
110
+ char invocation_str[UUID_STR_LEN];
111
+ uuid_unparse_lower(invocation_id.uuid, invocation_str);
112
+ sentry_set_tag("invocation_id", invocation_str);
113
79
- sentry_init(options);
114
+ // agent_events_version
115
+ sentry_set_tag("agent_events_version", TOSTRING(STATUS_FILE_VERSION));
116
}
117
82
-void nd_sentry_fini(void)
83
-{
118
+void nd_sentry_fini(void) {
119
if (!analytics_check_enabled())
120
return;
121
122
sentry_close();
123
}
124
90
-void nd_sentry_set_user(const char *guid)
91
-{
125
+void nd_sentry_set_user(const char *guid) {
126
sentry_value_t user = sentry_value_new_object();
127
sentry_value_set_by_key(user, "id", sentry_value_new_string(guid));
128
sentry_set_user(user);
129
}
130
97
-void nd_sentry_add_breadcrumb(const char *message)
98
-{
131
+void nd_sentry_add_fatal_message_as_breadcrumb(void) {
132
if (!analytics_check_enabled())
133
return;
134
102
- sentry_value_t crumb = sentry_value_new_breadcrumb("fatal", message);
135
+ const char *function = daemon_status_file_get_fatal_function();
136
+ if(!function || !*function)
137
+ function = "unknown";
138
+
139
+ // Set the transaction name to the function where the error occurred
140
+ // this should be low cardinality
141
+ sentry_set_transaction(function);
142
+
143
+ // Set the fingerprint to the function where the error occurred
144
+ sentry_set_fingerprint("{{ default }}", function, NULL);
145
+
146
+ sentry_value_t crumb = sentry_value_new_breadcrumb("fatal", "fatal() event details");
147
+
148
+ sentry_value_t data = sentry_value_new_object();
149
+ sentry_value_set_by_key(data, "message", sentry_value_new_string(daemon_status_file_get_fatal_message()));
150
+ sentry_value_set_by_key(data, "function", sentry_value_new_string(daemon_status_file_get_fatal_function()));
151
+ sentry_value_set_by_key(data, "filename", sentry_value_new_string(daemon_status_file_get_fatal_filename()));
152
+ sentry_value_set_by_key(data, "thread", sentry_value_new_string(daemon_status_file_get_fatal_thread()));
153
+
154
+ char line[UINT64_MAX_LENGTH];
155
+ print_uint64(line, daemon_status_file_get_fatal_line());
156
+ sentry_value_set_by_key(data, "line", sentry_value_new_string(line));
157
+ sentry_value_set_by_key(data, "errno", sentry_value_new_string(daemon_status_file_get_fatal_errno()));
158
+ sentry_value_set_by_key(data, "stack_trace", sentry_value_new_string(daemon_status_file_get_fatal_stack_trace()));
159
+
160
+ sentry_value_set_by_key(crumb, "data", data);
161
sentry_add_breadcrumb(crumb);
162
}
src/daemon/sentry-native/sentry-native.h
+3
-1
@@ -8,8 +8,10 @@
8
void nd_sentry_init(void);
9
void nd_sentry_fini(void);
10
11
+const char *nd_sentry_path(void);
12
+
13
void nd_sentry_set_user(const char *guid);
12
-void nd_sentry_add_breadcrumb(const char *message);
14
+void nd_sentry_add_fatal_message_as_breadcrumb(void);
15
16
void nd_sentry_crash_report(bool enable);
17
src/daemon/signal-handler.c
+3
@@ -155,6 +155,9 @@ void nd_cleanup_fatal_signals(void) {
155
if (sigaction(signals_waiting[i].signo, &act, NULL) == -1)
156
netdata_log_error("SIGNAL: Failed to cleanup signal handler for: %s", signals_waiting[i].name);
157
}
158
+
159
+ memset(original_handlers, 0, sizeof(original_handlers));
160
+ memset(original_sigactions, 0, sizeof(original_sigactions));
161
}
162
163
void nd_initialize_signals(bool chain_existing) {