@cryptotaxi247 / netdata-1 / commits / 6001d73c1

new exit cause: shutdown timeout (#19903)

* added new exit cause: shutdown timeout to daemon status and sentry * added to exit cause * increase status file version to 20 * tested; exit timeout now works independently of sentry * added timezone to status file * added agent_version_now when posting crash reports * added cloud provider, cloud instance and cloud region to status file and sentry

Costa Tsaousis committed Mar 19, 2025 at 19:11 UTC 6001d73c17498f89b2c34da9ce33b41ee1823696

8 files changed +155 -26

src/daemon/daemon-shutdown-watcher.c

+9 -7

@@ -3,6 +3,10 @@

3 #include "daemon-shutdown-watcher.h"

4 #include "daemon-status-file.h"

6 +#ifdef ENABLE_SENTRY

7 +#include "sentry-native/sentry-native.h"

8 +#endif

9 +

10 watcher_step_t *watcher_steps;

12 static struct completion shutdown_begin_completion;

@@ -12,6 +16,10 @@ static ND_THREAD *watcher_thread;

16 NEVER_INLINE

17 static void shutdown_timed_out(void) {

18 // keep this as a separate function, to have it logged like this in sentry

19 + daemon_status_file_shutdown_timeout();

20 +#ifdef ENABLE_SENTRY

21 + nd_sentry_add_shutdown_timeout_as_breadcrumb();

22 +#endif

23 abort();

24 }

@@ -48,7 +56,6 @@ static void watcher_wait_for_step(const watcher_step_id_t step_id, usec_t shutdo

57 daemon_status_file_shutdown_step(watcher_steps[step_id].msg);

51 -#ifdef ENABLE_SENTRY

59 // Wait with a timeout

60 time_t timeout = 135; // systemd gives us 150, we timeout at 135

@@ -57,11 +64,6 @@ static void watcher_wait_for_step(const watcher_step_id_t step_id, usec_t shutdo

64 remaining_seconds = 0;

66 bool ok = completion_timedwait_for(&watcher_steps[step_id].p, remaining_seconds);

60 -#else

61 - // Wait indefinitely

62 - bool ok = true;

63 - completion_wait_for(&watcher_steps[step_id].p);

64 -#endif

68 usec_t step_duration = now_monotonic_usec() - step_start_time;

@@ -191,7 +193,7 @@ void watcher_thread_start() {

193 completion_init(&shutdown_begin_completion);

194 completion_init(&shutdown_end_completion);

195

194 - watcher_thread = nd_thread_create("P[WATCHER]", NETDATA_THREAD_OPTION_JOINABLE, watcher_main, NULL);

196 + watcher_thread = nd_thread_create("EXIT_WATCHER", NETDATA_THREAD_OPTION_JOINABLE, watcher_main, NULL);

197 }

198

199 void watcher_thread_stop() {

src/daemon/daemon-status-file.c

+88 -17

@@ -211,6 +211,13 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

211 buffer_json_member_add_string_or_empty(wb, "container", ds->container);

212 buffer_json_member_add_time_t(wb, "uptime", ds->boottime);

213

214 + if(ds->v >= 20) {

215 + buffer_json_member_add_string_or_empty(wb, "timezone", ds->timezone);

216 + buffer_json_member_add_string_or_empty(wb, "cloud_provider", ds->cloud_provider_type);

217 + buffer_json_member_add_string_or_empty(wb, "cloud_instance", ds->cloud_instance_type);

218 + buffer_json_member_add_string_or_empty(wb, "cloud_region", ds->cloud_instance_region);

219 + }

220 +

221 buffer_json_member_add_object(wb, "boot");

222 {

223 buffer_json_member_add_uuid_compact(wb, "id", ds->boot_id.uuid);

@@ -323,6 +330,7 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

330 bool required_v16 = version >= 16 ? strict : false;

331 bool required_v17 = version >= 17 ? strict : false;

332 bool required_v18 = version >= 18 ? strict : false;

333 + bool required_v20 = version >= 20 ? strict : false;

334

335 // Parse timestamp

336 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required_v1);

@@ -419,6 +427,13 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

427 ds->var_cache = OS_SYSTEM_DISK_SPACE_EMPTY;

428 });

429 });

430 +

431 + if(version >= 20) {

432 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "timezone", ds->timezone, error, required_v20);

433 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "cloud_provider", ds->cloud_provider_type, error, required_v20);

434 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "cloud_instance", ds->cloud_instance_type, error, required_v20);

435 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "cloud_region", ds->cloud_instance_region, error, required_v20);

436 + }

437 });

438

439 // Parse os object

@@ -539,6 +554,10 @@ static void daemon_status_file_migrate_once(void) {

554 strncpyz(session_status.os_version, last_session_status.os_version, sizeof(session_status.os_version) - 1);

555 strncpyz(session_status.os_id, last_session_status.os_id, sizeof(session_status.os_id) - 1);

556 strncpyz(session_status.os_id_like, last_session_status.os_id_like, sizeof(session_status.os_id_like) - 1);

557 + strncpyz(session_status.timezone, last_session_status.timezone, sizeof(session_status.timezone) - 1);

558 + strncpyz(session_status.cloud_provider_type, last_session_status.cloud_provider_type, sizeof(session_status.cloud_provider_type) - 1);

559 + strncpyz(session_status.cloud_instance_type, last_session_status.cloud_instance_type, sizeof(session_status.cloud_instance_type) - 1);

560 + strncpyz(session_status.cloud_instance_region, last_session_status.cloud_instance_region, sizeof(session_status.cloud_instance_region) - 1);

561

562 session_status.restarts = last_session_status.restarts + 1;

563 session_status.reliability = last_session_status.reliability;

@@ -612,6 +631,9 @@ static void daemon_status_file_refresh(DAEMON_STATUS status) {

631

632 get_daemon_status_fields_from_system_info(&session_status);

633

634 + if(netdata_configured_timezone)

635 + strncpyz(session_status.timezone, netdata_configured_timezone, sizeof(session_status.timezone) - 1);

636 +

637 session_status.exit_reason = exit_initiated_get();

638 session_status.profile = nd_profile_detect_and_configure(false);

639

@@ -954,10 +976,11 @@ struct post_status_file_thread_data {

976 void post_status_file(struct post_status_file_thread_data *d) {

977 CLEAN_BUFFER *wb = buffer_create(0, NULL);

978 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);

957 - buffer_json_member_add_string(wb, "exit_cause", d->cause); // custom

958 - buffer_json_member_add_string(wb, "message", d->msg); // ECS

959 - buffer_json_member_add_uint64(wb, "priority", d->priority); // custom

960 - buffer_json_member_add_uint64(wb, "version_saved", d->status->v); // custom

979 + buffer_json_member_add_string(wb, "exit_cause", d->cause);

980 + buffer_json_member_add_string(wb, "message", d->msg);

981 + buffer_json_member_add_uint64(wb, "priority", d->priority);

982 + buffer_json_member_add_uint64(wb, "version_saved", d->status->v);

983 + buffer_json_member_add_string(wb, "agent_version_now", NETDATA_VERSION);

984 daemon_status_file_to_json(wb, d->status);

985 buffer_json_finalize(wb);

986

@@ -1157,6 +1180,11 @@ void daemon_status_file_check_crash(void) {

1180 msg = "Netdata was last crashed while exiting after receiving a deadly signal";

1181 pri = PRI_DEADLY_SIGNAL;

1182 }

1183 + else if(last_session_status.exit_reason & EXIT_REASON_SHUTDOWN_TIMEOUT) {

1184 + cause = "exit timeout";

1185 + msg = "Netdata was last killed because it couldn't shutdown on time";

1186 + pri = PRI_FATAL;

1187 + }

1188 else if(last_session_status.exit_reason != EXIT_REASON_NONE &&

1189 !is_exit_reason_normal(last_session_status.exit_reason)) {

1190 cause = "fatal on exit";

@@ -1425,6 +1453,46 @@ bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE c

1453 return duplicate;

1454 }

1455

1456 +// --------------------------------------------------------------------------------------------------------------------

1457 +// shutdown related functions

1458 +

1459 +static SPINLOCK shutdown_timeout_spinlock = SPINLOCK_INITIALIZER;

1460 +

1461 +void daemon_status_file_shutdown_timeout(void) {

1462 + FUNCTION_RUN_ONCE();

1463 +

1464 + spinlock_lock(&shutdown_timeout_spinlock);

1465 +

1466 + dsf_acquire(session_status);

1467 + exit_initiated_add(EXIT_REASON_SHUTDOWN_TIMEOUT);

1468 + session_status.exit_reason |= EXIT_REASON_SHUTDOWN_TIMEOUT;

1469 + dsf_release(session_status);

1470 +

1471 + strncpyz(session_status.fatal.function, "shutdown_timeout", sizeof(session_status.fatal.function) - 1);

1472 +

1473 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

1474 + daemon_status_file_save(wb, &session_status, false);

1475 +

1476 + // keep the spinlock locked, to prevent further steps updating the status

1477 +}

1478 +

1479 +void daemon_status_file_shutdown_step(const char *step) {

1480 + if(session_status.fatal.filename[0] || !spinlock_trylock(&shutdown_timeout_spinlock))

1481 + // we have a fatal logged

1482 + return;

1483 +

1484 + if(step != NULL)

1485 + snprintfz(session_status.fatal.function, sizeof(session_status.fatal.function), "shutdown(%s)", step);

1486 + else

1487 + session_status.fatal.function[0] = '\0';

1488 +

1489 + daemon_status_file_update_status(DAEMON_STATUS_EXITING);

1490 +

1491 + spinlock_unlock(&shutdown_timeout_spinlock);

1492 +}

1493 +

1494 +// --------------------------------------------------------------------------------------------------------------------

1495 +

1496 bool daemon_status_file_has_last_crashed(void) {

1497 return last_session_status.status != DAEMON_STATUS_EXITED || !is_exit_reason_normal(last_session_status.exit_reason);

1498 }

@@ -1449,19 +1517,6 @@ void daemon_status_file_startup_step(const char *step) {

1517 daemon_status_file_update_status(DAEMON_STATUS_INITIALIZING);

1518 }

1519

1452 -void daemon_status_file_shutdown_step(const char *step) {

1453 - if(session_status.fatal.filename[0])

1454 - // we have a fatal logged

1455 - return;

1456 -

1457 - if(step != NULL)

1458 - snprintfz(session_status.fatal.function, sizeof(session_status.fatal.function), "shutdown(%s)", step);

1459 - else

1460 - session_status.fatal.function[0] = '\0';

1461 -

1462 - daemon_status_file_update_status(DAEMON_STATUS_EXITING);

1463 -}

1464 -

1520 // --------------------------------------------------------------------------------------------------------------------

1521 // public API to get values

1522

@@ -1497,6 +1552,22 @@ const char *daemon_status_file_get_os_id_like(void) {

1552 return session_status.os_id_like;

1553 }

1554

1555 +const char *daemon_status_file_get_cloud_provider_type(void) {

1556 + return session_status.cloud_provider_type;

1557 +}

1558 +

1559 +const char *daemon_status_file_get_cloud_instance_type(void) {

1560 + return session_status.cloud_instance_type;

1561 +}

1562 +

1563 +const char *daemon_status_file_get_cloud_instance_region(void) {

1564 + return session_status.cloud_instance_region;

1565 +}

1566 +

1567 +const char *daemon_status_file_get_timezone(void) {

1568 + return session_status.timezone;

1569 +}

1570 +

1571 const char *daemon_status_file_get_fatal_filename(void) {

1572 return session_status.fatal.filename;

1573 }

src/daemon/daemon-status-file.h

+11 -2

@@ -7,7 +7,7 @@

7 #include "daemon/config/netdata-conf-profile.h"

8 #include "database/rrd-database-mode.h"

10 -#define STATUS_FILE_VERSION 19

10 +#define STATUS_FILE_VERSION 20

12 typedef enum {

13 DAEMON_STATUS_NONE,

@@ -73,6 +73,10 @@ typedef struct daemon_status_file {

73 char os_version[32]; // ECS: os.version

74 char os_id[64]; // ECS: os.family

75 char os_id_like[64]; // ECS: os.platform

76 + char timezone[32];

77 + char cloud_provider_type[32];

78 + char cloud_instance_type[32];

79 + char cloud_instance_region[32];

80 bool read_system_info;

82 char stack_traces[15]; // the backend for capturing stack traces

@@ -115,6 +119,7 @@ bool daemon_status_file_was_incomplete_shutdown(void);

119

120 void daemon_status_file_startup_step(const char *step);

121 void daemon_status_file_shutdown_step(const char *step);

122 +void daemon_status_file_shutdown_timeout(void);

123

124 void daemon_status_file_init(void);

125 void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);

@@ -127,8 +132,12 @@ const char *daemon_status_file_get_os_name(void);

132 const char *daemon_status_file_get_os_version(void);

133 const char *daemon_status_file_get_os_id(void);

134 const char *daemon_status_file_get_os_id_like(void);

135 +const char *daemon_status_file_get_timezone(void);

136 +const char *daemon_status_file_get_cloud_provider_type(void);

137 +const char *daemon_status_file_get_cloud_instance_type(void);

138 +const char *daemon_status_file_get_cloud_instance_region(void);

139

131 -const char *daemon_status_file_get_fatal_filename(void);

140 + const char *daemon_status_file_get_fatal_filename(void);

141 const char *daemon_status_file_get_fatal_function(void);

142 const char *daemon_status_file_get_fatal_message(void);

143 const char *daemon_status_file_get_fatal_errno(void);

src/daemon/sentry-native/sentry-native.c

+32

@@ -161,6 +161,10 @@ void nd_sentry_init(void) {

161 nd_sentry_set_tag("os_version", daemon_status_file_get_os_version());

162 nd_sentry_set_tag("os_id", daemon_status_file_get_os_id());

163 nd_sentry_set_tag("os_id_like", daemon_status_file_get_os_id_like());

164 + nd_sentry_set_tag("cloud_provider", daemon_status_file_get_cloud_provider_type());

165 + nd_sentry_set_tag("cloud_type", daemon_status_file_get_cloud_instance_type());

166 + nd_sentry_set_tag("cloud_region", daemon_status_file_get_cloud_instance_region());

167 + nd_sentry_set_tag("timezone", daemon_status_file_get_timezone());

168

169 // profile

170 CLEAN_BUFFER *profile = buffer_create(0, NULL);

@@ -262,3 +266,31 @@ void nd_sentry_add_fatal_message_as_breadcrumb(void) {

266 sentry_value_set_by_key(crumb, "data", data);

267 sentry_add_breadcrumb(crumb);

268 }

269 +

270 +void nd_sentry_add_shutdown_timeout_as_breadcrumb(void) {

271 + if (!analytics_check_enabled())

272 + return;

273 +

274 + const char *function = "shutdown_timeout";

275 + strncpyz(g_sentry_event_message, function, sizeof(g_sentry_event_message) - 1);

276 +

277 + nd_sentry_set_tag_uptime();

278 +

279 + // Set the transaction name to the function where the error occurred

280 + // this should be low cardinality

281 + sentry_set_transaction(function);

282 +

283 + // Set the fingerprint to the function where the error occurred

284 + sentry_set_fingerprint("{{ default }}", function, NULL);

285 +

286 + sentry_value_t crumb = sentry_value_new_breadcrumb("fatal", "shutdown_timeout() event details");

287 +

288 + sentry_value_t data = sentry_value_new_object();

289 + nd_sentry_add_key_value_charp(data, "function", function);

290 + nd_sentry_add_key_value_charp(data, "thread", nd_thread_tag());

291 + nd_sentry_add_key_value_uint64(data, "thread_id", gettid_cached());

292 + nd_sentry_add_key_value_charp(data, "status", DAEMON_STATUS_2str(daemon_status_file_get_status()));

293 +

294 + sentry_value_set_by_key(crumb, "data", data);

295 + sentry_add_breadcrumb(crumb);

296 +}

src/daemon/sentry-native/sentry-native.h

@@ -12,6 +12,7 @@ const char *nd_sentry_path(void);

13 void nd_sentry_set_user(const char *guid);

14 void nd_sentry_add_fatal_message_as_breadcrumb(void);

15 +void nd_sentry_add_shutdown_timeout_as_breadcrumb(void);

17 void nd_sentry_crash_report(bool enable);

src/database/rrdhost-system-info.c

@@ -659,5 +659,14 @@ void get_daemon_status_fields_from_system_info(DAEMON_STATUS_FILE *ds) {

659 ds->kubernetes = false;

660 }

661

662 + if(ri->cloud_provider_type && strcmp(ri->cloud_provider_type, "unknown") != 0)

663 + strncpyz(ds->cloud_provider_type, ri->cloud_provider_type, sizeof(ds->cloud_provider_type) - 1);

664 +

665 + if(ri->cloud_instance_type && strcmp(ri->cloud_instance_type, "unknown") != 0)

666 + strncpyz(ds->cloud_instance_type, ri->cloud_instance_type, sizeof(ds->cloud_instance_type) - 1);

667 +

668 + if(ri->cloud_instance_region && strcmp(ri->cloud_instance_region, "unknown") != 0)

669 + strncpyz(ds->cloud_instance_region, ri->cloud_instance_region, sizeof(ds->cloud_instance_region) - 1);

670 +

671 ds->read_system_info = true;

672 }

src/libnetdata/exit/exit_initiated.c

@@ -31,6 +31,7 @@ ENUM_STR_MAP_DEFINE(EXIT_REASON) = {

31 { EXIT_REASON_SYSTEM_SHUTDOWN, "system-shutdown"},

33 { EXIT_REASON_UPDATE, "update"},

34 + { EXIT_REASON_SHUTDOWN_TIMEOUT, "shutdown-timeout"},

36 // terminator

37 {0, NULL},

src/libnetdata/exit/exit_initiated.h

@@ -42,6 +42,9 @@ typedef enum {

43 // netdata update

44 EXIT_REASON_UPDATE = (1 << 18),

45 +

46 + // timeout while shutting down

47 + EXIT_REASON_SHUTDOWN_TIMEOUT = (1 << 19),

48 } EXIT_REASON;

50 #define EXIT_REASON_NORMAL \

@@ -74,6 +77,7 @@ typedef enum {

77 | EXIT_REASON_FATAL \

78 | EXIT_REASON_ALREADY_RUNNING \

79 | EXIT_REASON_OUT_OF_MEMORY \

80 + | EXIT_REASON_SHUTDOWN_TIMEOUT \

81 )

83 #define is_deadly_signal(reason) ((reason) & (EXIT_REASON_DEADLY_SIGNAL))