@cryptotaxi247 / netdata-1 / commits / 6f441d303

change log priorities on agent-events (#19880)

* change log priorities on agent-events * call abort() from dedicated functions to have proper sentry titles * invocation_id on sentry should be ephemeral_id be in compact form

Costa Tsaousis committed Mar 17, 2025 at 08:27 UTC 6f441d3038f8743dc869a874774e98a7bc74162b

6 files changed +58 -34

src/daemon/daemon-shutdown-watcher.c

+7 -1

@@ -9,6 +9,12 @@ static struct completion shutdown_begin_completion;

9 static struct completion shutdown_end_completion;

10 static ND_THREAD *watcher_thread;

12 +NEVER_INLINE

13 +static void shutdown_timed_out(void) {

14 + // keep this as a separate function, to have it logged like this in sentry

15 + abort();

16 +}

17 +

18 void watcher_shutdown_begin(void) {

19 completion_mark_complete(&shutdown_begin_completion);

20 }

@@ -87,7 +93,7 @@ static void watcher_wait_for_step(const watcher_step_id_t step_id, usec_t shutdo

93 #endif

95 daemon_status_file_shutdown_step("sentry timeout");

90 - abort();

96 + shutdown_timed_out();

97 }

98 }

src/daemon/daemon-shutdown.c

+10 -8

@@ -23,6 +23,15 @@ void abort_on_fatal_enable(void) {

23 abort_on_fatal = true;

24 }

26 +NEVER_INLINE

27 +static bool shutdown_on_fatal(void) {

28 + // keep this as a separate function, to have it logged like this in sentry

29 + if(abort_on_fatal)

30 + abort();

31 + else

32 + return false;

33 +}

34 +

35 void web_client_cache_destroy(void);

37 extern struct netdata_static_thread *static_threads;

@@ -360,10 +369,6 @@ void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char

369 fprintf(stderr, "WARNING: STRING has %zu strings still allocated.\n",

370 strings_referenced);

371

363 - // strings_destroy();

364 - // functions_destroy();

365 - // dyncfg_destroy();

366 -

372 fprintf(stderr, "All done, exiting...\n");

373 #endif

374

@@ -373,10 +378,7 @@ void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char

378 #endif

379

380 #ifdef ENABLE_SENTRY

376 - if (ret && abort_on_fatal) {

377 - abort();

378 - }

379 - else {

381 + if (!ret || !shutdown_on_fatal()) {

382 nd_sentry_fini();

383 curl_global_cleanup();

384 exit(ret);

src/daemon/daemon-status-file.c

+18 -15

@@ -952,10 +952,11 @@ struct log_priority {

952 ND_LOG_FIELD_PRIORITY post;

953 };

954

955 -struct log_priority PRI_ALL_NORMAL = { NDLP_NOTICE, NDLP_DEBUG };

956 -struct log_priority PRI_USER_SHOULD_FIX = { NDLP_WARNING, NDLP_INFO };

957 -struct log_priority PRI_NETDATA_BUG = { NDLP_CRIT, NDLP_ERR };

958 -struct log_priority PRI_BAD_BUT_NO_REASON = { NDLP_ERR, NDLP_WARNING };

955 +struct log_priority PRI_ALL_NORMAL = { NDLP_NOTICE, NDLP_DEBUG };

956 +struct log_priority PRI_USER_SHOULD_FIX = { NDLP_WARNING, NDLP_INFO };

957 +struct log_priority PRI_FATAL = { NDLP_ERR, NDLP_ERR };

958 +struct log_priority PRI_DEADLY_SIGNAL = { NDLP_CRIT, NDLP_CRIT };

959 +struct log_priority PRI_KILLED_HARD = { NDLP_ERR, NDLP_WARNING };

960

961 static bool is_ci(void) {

962 const char *ci = getenv("CI");

@@ -1021,14 +1022,14 @@ void daemon_status_file_check_crash(void) {

1022 else if(is_deadly_signal(last_session_status.exit_reason)) {

1023 cause = "deadly signal and exit";

1024 msg = "Netdata was last stopped gracefully after receiving a deadly signal";

1024 - pri = PRI_NETDATA_BUG;

1025 + pri = PRI_DEADLY_SIGNAL;

1026 this_is_a_crash = true;

1027 }

1028 else if(last_session_status.exit_reason != EXIT_REASON_NONE &&

1029 !is_exit_reason_normal(last_session_status.exit_reason)) {

1030 cause = "fatal and exit";

1031 msg = "Netdata was last stopped gracefully after it encountered a fatal error";

1031 - pri = PRI_NETDATA_BUG;

1032 + pri = PRI_FATAL;

1033 this_is_a_crash = true;

1034 }

1035 else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {

@@ -1062,7 +1063,7 @@ void daemon_status_file_check_crash(void) {

1063 else if(is_deadly_signal(last_session_status.exit_reason)) {

1064 cause = "deadly signal on start";

1065 msg = "Netdata was last crashed while starting after receiving a deadly signal";

1065 - pri = PRI_NETDATA_BUG;

1066 + pri = PRI_DEADLY_SIGNAL;

1067 this_is_a_crash = true;

1068 }

1069 else if (last_session_status.exit_reason & EXIT_REASON_OUT_OF_MEMORY) {

@@ -1097,12 +1098,12 @@ void daemon_status_file_check_crash(void) {

1098 !is_exit_reason_normal(last_session_status.exit_reason)) {

1099 cause = "fatal on start";

1100 msg = "Netdata was last crashed while starting, because of a fatal error";

1100 - pri = PRI_NETDATA_BUG;

1101 + pri = PRI_FATAL;

1102 }

1103 else {

1104 cause = "killed hard on start";

1105 msg = "Netdata was last killed/crashed while starting";

1105 - pri = PRI_BAD_BUT_NO_REASON;

1106 + pri = PRI_KILLED_HARD;

1107 }

1108 this_is_a_crash = true;

1109 break;

@@ -1111,27 +1112,29 @@ void daemon_status_file_check_crash(void) {

1112 if(is_deadly_signal(last_session_status.exit_reason)) {

1113 cause = "deadly signal on exit";

1114 msg = "Netdata was last crashed while exiting after receiving a deadly signal";

1114 - pri = PRI_NETDATA_BUG;

1115 - this_is_a_crash = true;

1115 + pri = PRI_DEADLY_SIGNAL;

1116 }

1117 else if(last_session_status.exit_reason != EXIT_REASON_NONE &&

1118 !is_exit_reason_normal(last_session_status.exit_reason)) {

1119 cause = "fatal on exit";

1120 msg = "Netdata was last killed/crashed while exiting after encountering an error";

1121 + pri = PRI_FATAL;

1122 }

1123 else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {

1124 cause = "killed hard on shutdown";

1125 msg = "Netdata was last killed/crashed while exiting due to system shutdown";

1126 + pri = PRI_KILLED_HARD;

1127 }

1128 else if(new_version || (last_session_status.exit_reason & EXIT_REASON_UPDATE)) {

1129 cause = "killed hard on update";

1130 msg = "Netdata was last killed/crashed while exiting to update to a new version";

1131 + pri = PRI_KILLED_HARD;

1132 }

1133 else {

1134 cause = "killed hard on exit";

1135 msg = "Netdata was last killed/crashed while it was instructed to exit";

1136 + pri = PRI_KILLED_HARD;

1137 }

1134 - pri = PRI_NETDATA_BUG;

1138 this_is_a_crash = true;

1139 break;

1140

@@ -1152,19 +1155,19 @@ void daemon_status_file_check_crash(void) {

1155 else if(is_deadly_signal(last_session_status.exit_reason)) {

1156 cause = "deadly signal";

1157 msg = "Netdata was last crashed after receiving a deadly signal";

1155 - pri = PRI_NETDATA_BUG;

1158 + pri = PRI_DEADLY_SIGNAL;

1159 this_is_a_crash = true;

1160 }

1161 else if (last_session_status.exit_reason != EXIT_REASON_NONE &&

1162 !is_exit_reason_normal(last_session_status.exit_reason)) {

1163 cause = "killed fatal";

1164 msg = "Netdata was last crashed due to a fatal error";

1162 - pri = PRI_NETDATA_BUG;

1165 + pri = PRI_FATAL;

1166 }

1167 else {

1168 cause = "killed hard";

1169 msg = "Netdata was last killed/crashed while operating normally";

1167 - pri = PRI_BAD_BUT_NO_REASON;

1170 + pri = PRI_KILLED_HARD;

1171 this_is_a_crash = true;

1172 }

1173 break;

src/daemon/sentry-native/sentry-native.c

+3 -3

@@ -107,9 +107,9 @@ void nd_sentry_init(void) {

107

108 // invocation_id

109 ND_UUID invocation_id = nd_log_get_invocation_id();

110 - char invocation_str[UUID_STR_LEN];

111 - uuid_unparse_lower(invocation_id.uuid, invocation_str);

112 - sentry_set_tag("invocation_id", invocation_str);

110 + char invocation_str[UUID_COMPACT_STR_LEN];

111 + uuid_unparse_lower_compact(invocation_id.uuid, invocation_str);

112 + sentry_set_tag("ephemeral_id", invocation_str);

113

114 // agent_events_version

115 sentry_set_tag("agent_events_version", TOSTRING(STATUS_FILE_VERSION));

src/libnetdata/common.h

@@ -342,6 +342,7 @@ typedef uint32_t uid_t;

342 #define ALWAYS_INLINE_HOT inline __attribute__((hot, always_inline)) // Encourages optimization and forces inlining

343 #define ALWAYS_INLINE_HOT_FLATTEN inline __attribute__((hot, always_inline, flatten)) // Encourages optimization and forces inlining and flattening

344 #define NOT_INLINE_HOT __attribute__((hot)) // Encourages optimization but doesn’t force inlining.

345 +#define NEVER_INLINE __attribute__((noinline))

346 #else

347 #define UNUSED_FUNCTION(x) UNUSED_##x

348 #define ALWAYS_INLINE_ONLY

@@ -349,6 +350,7 @@ typedef uint32_t uid_t;

350 #define ALWAYS_INLINE_HOT inline

351 #define ALWAYS_INLINE_HOT_FLATTEN inline

352 #define NOT_INLINE_HOT

353 +#define NEVER_INLINE

354 #endif

355

356 // --------------------------------------------------------------------------------------------------------------------

src/libnetdata/log/nd_log.c

+18 -7

@@ -458,6 +458,22 @@ void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_F

458 erl->count = 0;

459 }

460

461 +NEVER_INLINE NORETURN

462 +static void recursive_fatal_abort(void) {

463 + // keep this as a separate function, to have it logged like this in sentry

464 +#ifdef ENABLE_SENTRY

465 + abort();

466 +#endif

467 + _exit(1);

468 +}

469 +

470 +NEVER_INLINE NORETURN

471 +static void fatal_abort_internal_checks(void) {

472 + // keep this as a separate function, to have it logged like this in sentry

473 + abort();

474 + _exit(1);

475 +}

476 +

477 void netdata_logger_fatal(const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {

478 static size_t already_in_fatal = 0;

479

@@ -468,12 +484,7 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned

484 fprintf(stderr, "\nRECURSIVE FATAL STATEMENTS, latest from %s() of %lu@%s, EXITING NOW! 23e93dfccbf64e11aac858b9410d8a82\n",

485 function, line, file);

486 fflush(stderr);

471 -

472 -#ifdef ENABLE_SENTRY

473 - abort();

474 -#else

475 - _exit(1);

476 -#endif

487 + recursive_fatal_abort();

488 }

489

490 // send this event to deamon_status_file

@@ -521,7 +532,7 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned

532 snprintfz(action_result, 60, "%s:%s:%s", program_name, tag_to_send, function);

533

534 #ifdef NETDATA_INTERNAL_CHECKS

524 - abort();

535 + fatal_abort_internal_checks();

536 #endif

537

538 if(nd_log.fatal_final_cb)