@cryptotaxi247 / netdata-1 / commits / 619dda24c

daemon status 24d (#20011)

* add the shutdown timings to status files stack trace * count the unique number of crashes ever

Costa Tsaousis committed Mar 31, 2025 at 19:22 UTC 619dda24cba36345dc60fda5d6479bcfacef9e8f

4 files changed +24 -5

src/daemon/daemon-shutdown-watcher.c

+10 -2

@@ -13,10 +13,12 @@ static struct completion shutdown_begin_completion;

13 static struct completion shutdown_end_completion;

14 static ND_THREAD *watcher_thread;

16 +static BUFFER *steps_timings = NULL;

17 +

18 NEVER_INLINE

19 static void shutdown_timed_out(void) {

20 // keep this as a separate function, to have it logged like this in sentry

19 - daemon_status_file_shutdown_timeout();

21 + daemon_status_file_shutdown_timeout(steps_timings);

22 #ifdef ENABLE_SENTRY

23 nd_sentry_add_shutdown_timeout_as_breadcrumb();

24 #endif

@@ -37,6 +39,11 @@ void watcher_step_complete(watcher_step_id_t step_id) {

40 static void watcher_wait_for_step(const watcher_step_id_t step_id, usec_t shutdown_start_time)

41 {

42 + if(!steps_timings) {

43 + steps_timings = buffer_create(0, NULL);

44 + buffer_strcat(steps_timings, "# shutdown steps timings");

45 + }

46 +

47 usec_t step_start_time = now_monotonic_usec();

48 usec_t step_start_duration = step_start_time - shutdown_start_time;

@@ -71,6 +78,8 @@ static void watcher_wait_for_step(const watcher_step_id_t step_id, usec_t shutdo

78 duration_snprintf(

79 step_duration_txt, sizeof(step_duration_txt), (int64_t)(step_duration), "us", true);

81 + buffer_sprintf(steps_timings, "\n#%u '%s': %s", step_id + 1, watcher_steps[step_id].msg, step_duration_txt);

82 +

83 if (ok) {

84 netdata_log_info("shutdown step: [%d/%d] - {at %s} finished '%s' in %s",

85 (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, start_duration_txt,

@@ -94,7 +103,6 @@ static void watcher_wait_for_step(const watcher_step_id_t step_id, usec_t shutdo

103 watcher_steps[step_id].msg, step_duration_txt);

104 #endif

105

97 - daemon_status_file_shutdown_step("sentry timeout");

106 shutdown_timed_out();

107 }

108 }

src/daemon/status-file.c

+11 -1

@@ -112,6 +112,9 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

112 buffer_json_member_add_uuid(wb, "claim_id", ds->claim_id.uuid);

113 buffer_json_member_add_uint64(wb, "restarts", ds->restarts);

114

115 + if(ds->v >= 24)

116 + buffer_json_member_add_uint64(wb, "crashes", ds->crashes);

117 +

118 if(ds->v >= 22) {

119 buffer_json_member_add_uint64(wb, "posts", ds->posts);

120 buffer_json_member_add_string(wb, "aclk", CLOUD_STATUS_2str(ds->cloud_status));

@@ -316,6 +319,9 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

319 if(version >= 4)

320 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, restarts_key, ds->restarts, error, required_v4);

321

322 + if(version >= 24)

323 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "crashes", ds->crashes, error, required_v24);

324 +

325 if(version >= 22) {

326 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "posts", ds->posts, error, required_v22);

327 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "aclk", CLOUD_STATUS_2id, ds->cloud_status, error, required_v22);

@@ -492,9 +498,11 @@ static void daemon_status_file_migrate_once(void) {

498

499 session_status.posts = last_session_status.posts;

500 session_status.restarts = last_session_status.restarts + 1;

501 + session_status.crashes = last_session_status.crashes;

502 session_status.reliability = last_session_status.reliability;

503

504 if(daemon_status_file_has_last_crashed(&last_session_status)) {

505 + session_status.crashes++;

506 if(session_status.reliability > 0) session_status.reliability = 0;

507 session_status.reliability--;

508 }

@@ -1228,7 +1236,7 @@ bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE c

1236

1237 static SPINLOCK shutdown_timeout_spinlock = SPINLOCK_INITIALIZER;

1238

1231 -void daemon_status_file_shutdown_timeout(void) {

1239 +void daemon_status_file_shutdown_timeout(BUFFER *trace) {

1240 FUNCTION_RUN_ONCE();

1241

1242 spinlock_lock(&shutdown_timeout_spinlock);

@@ -1236,6 +1244,8 @@ void daemon_status_file_shutdown_timeout(void) {

1244 dsf_acquire(session_status);

1245 exit_initiated_add(EXIT_REASON_SHUTDOWN_TIMEOUT);

1246 session_status.exit_reason |= EXIT_REASON_SHUTDOWN_TIMEOUT;

1247 + if(trace && buffer_strlen(trace) && stack_trace_is_empty(&session_status))

1248 + strncpyz(session_status.fatal.stack_trace, buffer_tostring(trace), sizeof(session_status.fatal.stack_trace) - 1);

1249 dsf_release(session_status);

1250

1251 strncpyz(session_status.fatal.function, "shutdown_timeout", sizeof(session_status.fatal.function) - 1);

src/daemon/status-file.h

+2 -1

@@ -48,6 +48,7 @@ typedef struct daemon_status_file {

48 time_t uptime; // netdata uptime

49 usec_t timestamp_ut; // the timestamp of the status file

50 size_t restarts; // the number of times this agent has restarted (ever)

51 + size_t crashes; // the number of times this agent has crashed (ever)

52 size_t posts; // the number of posts to the backend

53 ssize_t reliability; // consecutive restarts: > 0 reliable, < 0 crashing

@@ -123,7 +124,7 @@ bool daemon_status_file_was_incomplete_shutdown(void);

124

125 void daemon_status_file_startup_step(const char *step);

126 void daemon_status_file_shutdown_step(const char *step);

126 -void daemon_status_file_shutdown_timeout(void);

127 +void daemon_status_file_shutdown_timeout(BUFFER *trace);

128

129 void daemon_status_file_init(void);

130 void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);

src/libnetdata/log/nd_log-stacktrace.c

+1 -1

@@ -4,7 +4,7 @@

5 bool nd_log_forked = false;

7 -#define NO_STACK_TRACE_PREFIX "stack trace not available: "

7 +#define NO_STACK_TRACE_PREFIX "info: stack trace is not available, "

9 #if defined(HAVE_LIBBACKTRACE)

10 #include "backtrace-supported.h"