@cryptotaxi247 / netdata-1 / commits / 8c2443695

daemon status 22 (#19953)

* annotate crash and healthy loops * log successful and failed posts to agent-events * move the already running fatal() outside of netdata_main() * count the number of posts made to the backend * added more info to crash reports init step, to find where the killed hard is happening * add aclk status to daemon status file * also check for localhost * remove the cloud "connecting" state * avoid duplicates

Costa Tsaousis committed Mar 24, 2025 at 19:06 UTC 8c244369582fcd4b78dd9081d14fb33a653a32c5

12 files changed +110 -64

src/aclk/aclk.c

@@ -30,12 +30,16 @@ mqtt_wss_client mqttwss_client;

30 static bool aclk_connected = false;

31 static inline void aclk_set_connected(void) {

32 __atomic_store_n(&aclk_connected, true, __ATOMIC_RELAXED);

33 +

34 + daemon_status_file_update_status(DAEMON_STATUS_NONE);

35 }

36 static inline void aclk_set_disconnected(void) {

37 __atomic_store_n(&aclk_connected, false, __ATOMIC_RELAXED);

39 if(mqttwss_client)

40 mqtt_wss_reset_stats(mqttwss_client);

41 +

42 + daemon_status_file_update_status(DAEMON_STATUS_NONE);

43 }

45 inline bool aclk_online(void) {

src/claim/cloud-status.c

+16 -30

@@ -2,41 +2,27 @@

3 #include "claim.h"

5 -const char *cloud_status_to_string(CLOUD_STATUS status) {

6 - switch(status) {

7 - default:

8 - case CLOUD_STATUS_AVAILABLE:

9 - return "available";

10 -

11 - case CLOUD_STATUS_BANNED:

12 - return "banned";

13 -

14 - case CLOUD_STATUS_OFFLINE:

15 - return "offline";

16 -

17 - case CLOUD_STATUS_ONLINE:

18 - return "online";

19 -

20 - case CLOUD_STATUS_CONNECTING:

21 - return "connecting";

22 -

23 - case CLOUD_STATUS_INDIRECT:

24 - return "indirect";

25 - }

26 -}

5 +ENUM_STR_MAP_DEFINE(CLOUD_STATUS) = {

6 + { CLOUD_STATUS_ONLINE, "online"},

7 + { CLOUD_STATUS_INDIRECT, "indirect"},

8 + { CLOUD_STATUS_AVAILABLE, "available"},

9 + { CLOUD_STATUS_BANNED, "banned"},

10 + { CLOUD_STATUS_OFFLINE, "offline"},

11 +

12 + // terminator

13 + { 0, NULL },

14 +};

15 +ENUM_STR_DEFINE_FUNCTIONS(CLOUD_STATUS, CLOUD_STATUS_AVAILABLE, "available");

17 CLOUD_STATUS cloud_status(void) {

18 if(unlikely(aclk_disable_runtime))

19 return CLOUD_STATUS_BANNED;

32 - if(likely(aclk_online())) {

33 - if (rrdhost_flag_check(localhost, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS))

34 - return CLOUD_STATUS_ONLINE;

35 - else

36 - return CLOUD_STATUS_CONNECTING;

37 - }

21 + if(likely(aclk_online()))

22 + return CLOUD_STATUS_ONLINE;

39 - if(localhost->sender &&

24 + if(localhost &&

25 + localhost->sender &&

26 rrdhost_flag_check(localhost, RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS) &&

27 stream_sender_has_capabilities(localhost, STREAM_CAP_NODE_ID) &&

28 !UUIDiszero(localhost->node_id) &&

@@ -83,7 +69,7 @@ CLOUD_STATUS buffer_json_cloud_status(BUFFER *wb, time_t now_s) {

69 time_t last_change = cloud_last_change();

70 time_t next_connect = cloud_next_connection_attempt();

71 buffer_json_member_add_uint64(wb, "id", id);

86 - buffer_json_member_add_string(wb, "status", cloud_status_to_string(status));

72 + buffer_json_member_add_string(wb, "status", CLOUD_STATUS_2str(status));

73 buffer_json_member_add_time_t(wb, "since", last_change);

74 buffer_json_member_add_time_t(wb, "age", now_s - last_change);

src/claim/cloud-status.h

+4 -3

@@ -3,18 +3,19 @@

3 #ifndef NETDATA_CLOUD_STATUS_H

4 #define NETDATA_CLOUD_STATUS_H

6 -#include "daemon/common.h"

6 +#include "libnetdata/libnetdata.h"

7 +#include "claim/cloud-status.h"

9 typedef enum __attribute__((packed)) {

10 CLOUD_STATUS_AVAILABLE = 1, // cloud and aclk functionality is available, but the agent is not claimed

11 CLOUD_STATUS_BANNED, // the agent has been banned from cloud

12 CLOUD_STATUS_OFFLINE, // the agent tries to connect to cloud, but cannot do it

13 CLOUD_STATUS_INDIRECT, // the agent is connected to cloud via a parent

13 - CLOUD_STATUS_CONNECTING, // the agent is connecting

14 CLOUD_STATUS_ONLINE, // the agent is connected to cloud

15 } CLOUD_STATUS;

17 -const char *cloud_status_to_string(CLOUD_STATUS status);

17 +ENUM_STR_DEFINE_FUNCTIONS_EXTERN(CLOUD_STATUS);

18 +

19 CLOUD_STATUS cloud_status(void);

21 time_t cloud_last_change(void);

src/daemon/daemon-status-file.c

+66 -8

@@ -174,6 +174,11 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

174 buffer_json_member_add_uuid(wb, "claim_id", ds->claim_id.uuid);

175 buffer_json_member_add_uint64(wb, "restarts", ds->restarts);

176

177 + if(ds->v >= 22) {

178 + buffer_json_member_add_uint64(wb, "posts", ds->posts);

179 + buffer_json_member_add_string(wb, "aclk", CLOUD_STATUS_2str(ds->cloud_status));

180 + }

181 +

182 ND_PROFILE_2json(wb, "profile", ds->profile);

183 buffer_json_member_add_string(wb, "status", DAEMON_STATUS_2str(ds->status));

184 EXIT_REASON_2json(wb, "exit_reason", ds->exit_reason);

@@ -337,6 +342,7 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

342 bool required_v18 = version >= 18 ? strict : false;

343 bool required_v20 = version >= 20 ? strict : false;

344 bool required_v21 = version >= 21 ? strict : false;

345 + bool required_v22 = version >= 22 ? strict : false;

346

347 // Parse timestamp

348 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required_v1);

@@ -378,6 +384,11 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

384 if(version >= 4)

385 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, restarts_key, ds->restarts, error, required_v4);

386

387 + if(version >= 22) {

388 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "posts", ds->posts, error, required_v22);

389 + JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "aclk", CLOUD_STATUS_2id, ds->cloud_status, error, required_v22);

390 + }

391 +

392 if(version >= 14) {

393 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, db_mode_key, rrd_memory_mode_id, ds->db_mode, error, required_v14);

394 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, db_tiers_key, ds->db_tiers, error, required_v14);

@@ -569,10 +580,11 @@ static void daemon_status_file_migrate_once(void) {

580 strncpyz(session_status.cloud_instance_type, last_session_status.cloud_instance_type, sizeof(session_status.cloud_instance_type) - 1);

581 strncpyz(session_status.cloud_instance_region, last_session_status.cloud_instance_region, sizeof(session_status.cloud_instance_region) - 1);

582

583 + session_status.posts = last_session_status.posts;

584 session_status.restarts = last_session_status.restarts + 1;

585 session_status.reliability = last_session_status.reliability;

586

575 - if(daemon_status_file_has_last_crashed()) {

587 + if(daemon_status_file_has_last_crashed(&last_session_status)) {

588 if(session_status.reliability > 0) session_status.reliability = 0;

589 session_status.reliability--;

590 }

@@ -627,6 +639,7 @@ static void daemon_status_file_refresh(DAEMON_STATUS status) {

639 session_status.invocation = nd_log_get_invocation_id();

640 session_status.db_mode = default_rrd_memory_mode;

641 session_status.db_tiers = nd_profile.storage_tiers;

642 + session_status.cloud_status = cloud_status();

643

644 session_status.oom_protection = dbengine_out_of_memory_protection;

645 session_status.netdata_max_rss = process_max_rss();

@@ -990,7 +1003,34 @@ struct post_status_file_thread_data {

1003 DAEMON_STATUS_FILE *status;

1004 };

1005

993 -void post_status_file(struct post_status_file_thread_data *d) {

1006 +static const char *agent_health(DAEMON_STATUS_FILE *ds) {

1007 + if(daemon_status_file_has_last_crashed(ds)) {

1008 + // it crashed

1009 +

1010 + if(ds->restarts == 1)

1011 + return "crash-first";

1012 + else if(ds->reliability <= -2)

1013 + return "crash-loop";

1014 + else if(ds->reliability < 0)

1015 + return "crash-repeated";

1016 + else

1017 + return "crash-entered";

1018 + }

1019 +

1020 + // it didn't crash

1021 + if(ds->restarts == 1)

1022 + return "healthy-first";

1023 + else if(ds->reliability >= 2)

1024 + return "healthy-loop";

1025 + else if(ds->reliability > 0)

1026 + return "healthy-repeated";

1027 + else

1028 + return "healthy-recovered";

1029 +}

1030 +

1031 +static void post_status_file(struct post_status_file_thread_data *d) {

1032 + daemon_status_file_startup_step("startup(crash reports json)");

1033 +

1034 CLEAN_BUFFER *wb = buffer_create(0, NULL);

1035 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);

1036 buffer_json_member_add_string(wb, "exit_cause", d->cause);

@@ -1001,6 +1041,7 @@ void post_status_file(struct post_status_file_thread_data *d) {

1041 buffer_json_member_add_boolean(wb, "host_memory_critical",

1042 OS_SYSTEM_MEMORY_OK(d->status->memory) && d->status->memory.ram_available_bytes <= d->status->oom_protection);

1043 buffer_json_member_add_uint64(wb, "host_memory_free_percent", (uint64_t)round(os_system_memory_available_percent(d->status->memory)));

1044 + buffer_json_member_add_string(wb, "agent_health", agent_health(d->status));

1045 daemon_status_file_to_json(wb, d->status);

1046 buffer_json_finalize(wb);

1047

@@ -1010,6 +1051,8 @@ void post_status_file(struct post_status_file_thread_data *d) {

1051 if(!curl)

1052 return;

1053

1054 + daemon_status_file_startup_step("startup(crash reports curl)");

1055 +

1056 curl_easy_setopt(curl, CURLOPT_URL, "https://agent-events.netdata.cloud/agent-events");

1057 curl_easy_setopt(curl, CURLOPT_POST, 1L);

1058 curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_data);

@@ -1020,10 +1063,17 @@ void post_status_file(struct post_status_file_thread_data *d) {

1063

1064 CURLcode rc = curl_easy_perform(curl);

1065 if(rc == CURLE_OK) {

1066 + daemon_status_file_startup_step("startup(crash reports dedup)");

1067 + nd_log(NDLS_DAEMON, NDLP_INFO, "Posted last status to agent-events successfully.");

1068 uint64_t hash = daemon_status_file_hash(d->status, d->msg, d->cause);

1069 dedup_keep_hash(&session_status, hash, false);

1070 + session_status.posts++;

1071 daemon_status_file_save(wb, &session_status, true);

1072 }

1073 + else

1074 + nd_log(NDLS_DAEMON, NDLP_INFO, "Failed to post last status to agent-events.");

1075 +

1076 + daemon_status_file_startup_step("startup(crash reports cleanup)");

1077

1078 curl_easy_cleanup(curl);

1079 curl_slist_free_all(headers);

@@ -1311,18 +1361,24 @@ void daemon_status_file_check_crash(void) {

1361 "Last exit status: %s (%s):\n\n%s",

1362 NETDATA_VERSION, msg, cause, buffer_tostring(wb));

1363

1364 + daemon_status_file_startup_step("startup(crash reports check)");

1365 +

1366 enum crash_report_t r = check_crash_reports_config();

1367 if( // must be first for netdata.conf option to be used

1368 (r == DSF_REPORT_ALL || (this_is_a_crash && r == DSF_REPORT_CRASHES)) &&

1369

1318 - // we have a previous status, or

1319 - // (we managed to save the current one, and (we have more than 2 restarts, or this is not a CI run))

1320 - (!no_previous_status || (daemon_status_file_saved && (last_session_status.restarts > 2 || !is_ci()))) &&

1370 + // we have a previous status, or we managed to save the current one

1371 + (!no_previous_status || daemon_status_file_saved) &&

1372 +

1373 + // we have more than 2 restarts, or this is not a CI run

1374 + (last_session_status.restarts > 2 || !is_ci()) &&

1375

1376 // we have not reported this

1377 !dedup_already_posted(&session_status, daemon_status_file_hash(&last_session_status, msg, cause), false)

1378

1379 ) {

1380 + daemon_status_file_startup_step("startup(crash reports prep)");

1381 +

1382 netdata_conf_ssl();

1383

1384 if(no_previous_status) {

@@ -1545,9 +1601,11 @@ void daemon_status_file_shutdown_step(const char *step) {

1601

1602 // --------------------------------------------------------------------------------------------------------------------

1603

1548 -bool daemon_status_file_has_last_crashed(void) {

1549 - return (last_session_status.status != DAEMON_STATUS_NONE && last_session_status.status != DAEMON_STATUS_EXITED) ||

1550 - !is_exit_reason_normal(last_session_status.exit_reason);

1604 +bool daemon_status_file_has_last_crashed(DAEMON_STATUS_FILE *ds) {

1605 + if(!ds) ds = &last_session_status;

1606 +

1607 + return (ds->status != DAEMON_STATUS_NONE && ds->status != DAEMON_STATUS_EXITED) ||

1608 + !is_exit_reason_normal(ds->exit_reason);

1609 }

1610

1611 bool daemon_status_file_was_incomplete_shutdown(void) {

src/daemon/daemon-status-file.h

+5 -2

@@ -6,8 +6,9 @@

6 #include "libnetdata/libnetdata.h"

7 #include "daemon/config/netdata-conf-profile.h"

8 #include "database/rrd-database-mode.h"

9 +#include "claim/cloud-status.h"

10 -#define STATUS_FILE_VERSION 21

11 +#define STATUS_FILE_VERSION 22

13 typedef enum {

14 DAEMON_STATUS_NONE,

@@ -37,6 +38,7 @@ typedef struct daemon_status_file {

38 ND_PROFILE profile; // the profile of the agent

39 DAEMON_OS_TYPE os_type;

40 RRD_DB_MODE db_mode;

41 + CLOUD_STATUS cloud_status;

42 uint8_t db_tiers;

43 bool kubernetes;

44 bool sentry_available; // true when sentry support is compiled in

@@ -45,6 +47,7 @@ typedef struct daemon_status_file {

47 time_t uptime; // netdata uptime

48 usec_t timestamp_ut; // the timestamp of the status file

49 size_t restarts; // the number of times this agent has restarted (ever)

50 + size_t posts; // the number of posts to the backend

51 ssize_t reliability; // consecutive restarts: > 0 reliable, < 0 crashing

53 ND_UUID boot_id; // the boot id of the system

@@ -116,7 +119,7 @@ bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE c

119 // check for a crash

120 void daemon_status_file_check_crash(void);

121

119 -bool daemon_status_file_has_last_crashed(void);

122 +bool daemon_status_file_has_last_crashed(DAEMON_STATUS_FILE *ds);

123 bool daemon_status_file_was_incomplete_shutdown(void);

124

125 void daemon_status_file_startup_step(const char *step);

src/daemon/main.c

+3 -10

@@ -922,15 +922,8 @@ int netdata_main(int argc, char **argv) {

922 web_server_threading_selection();

923

924 delta_startup_time("web server sockets");

925 - if(web_server_mode != WEB_SERVER_MODE_NONE) {

926 - errno_clear();

927 - if (!api_listen_sockets_setup()) {

928 - exit_initiated_add(EXIT_REASON_ALREADY_RUNNING);

929 - daemon_status_file_update_status(DAEMON_STATUS_NONE);

930 - fatal("Cannot setup listen port(s). Is Netdata already running?");

931 - exit(1);

932 - }

933 - }

925 + if(web_server_mode != WEB_SERVER_MODE_NONE)

926 + web_server_listen_sockets_setup();

927

928 // ----------------------------------------------------------------------------------------------------------------

929 delta_startup_time("sqlite");

@@ -1105,7 +1098,7 @@ int netdata_main(int argc, char **argv) {

1098

1099 analytics_statistic_t start_statistic = {"START", "-", "-"};

1100 analytics_statistic_send(&start_statistic);

1108 - if (daemon_status_file_has_last_crashed()) {

1101 + if (daemon_status_file_has_last_crashed(NULL)) {

1102 analytics_statistic_t crash_statistic = {"CRASH", "-", "-"};

1103 analytics_statistic_send(&crash_statistic);

1104 }

src/daemon/sentry-native/sentry-native.c

+1 -1

@@ -46,7 +46,7 @@ static void nd_sentry_set_tag_uint64(const char *key, uint64_t value) {

46 nd_sentry_set_tag(key, buf);

47 }

49 -static void nd_sentry_set_tag_uuid(const char *key, const ND_UUID uuid) {

49 +static inline void nd_sentry_set_tag_uuid(const char *key, const ND_UUID uuid) {

50 if(UUIDiszero(uuid))

51 return;

src/libnetdata/log/nd_log-stacktrace.c

+1 -1

@@ -97,7 +97,7 @@ static void bt_error_handler(void *data, const char *msg, int errnum) {

97 }

99 // Full callback for libbacktrace

100 -static int bt_full_handler(void *data, uintptr_t pc,

100 +static int bt_full_handler(void *data, uintptr_t pc __maybe_unused,

101 const char *filename, int lineno,

102 const char *function) {

103 backtrace_data_t *bt_data = (backtrace_data_t *)data;

src/registry/registry.c

+1 -1

@@ -183,7 +183,7 @@ int registry_request_hello_json(RRDHOST *host, struct web_client *w, bool do_not

183 buffer_json_object_close(w->response.data);

184

185 CLOUD_STATUS status = cloud_status();

186 - buffer_json_member_add_string(w->response.data, "cloud_status", cloud_status_to_string(status));

186 + buffer_json_member_add_string(w->response.data, "cloud_status", CLOUD_STATUS_2str(status));

187 buffer_json_member_add_string(w->response.data, "cloud_base_url", registry.cloud_base_url);

188

189 buffer_json_member_add_string(w->response.data, "registry", registry.registry_to_announce);

src/web/api/v2/api_v2_claim.c

-1

@@ -90,7 +90,6 @@ static bool agent_can_be_claimed(void) {

91 case CLOUD_STATUS_BANNED:

92 case CLOUD_STATUS_ONLINE:

93 - case CLOUD_STATUS_CONNECTING:

93 return false;

94 }

src/web/server/web_server.c

+8 -6

@@ -57,16 +57,18 @@ void debug_sockets() {

57 buffer_free(wb);

58 }

60 -bool api_listen_sockets_setup(void) {

61 - int socks = listen_sockets_setup(&api_sockets);

60 +void web_server_listen_sockets_setup(void) {

61 + errno_clear();

63 - if(!socks)

64 - return false;

63 + int socks = listen_sockets_setup(&api_sockets);

64 + if(!socks) {

65 + exit_initiated_add(EXIT_REASON_ALREADY_RUNNING);

66 + daemon_status_file_update_status(DAEMON_STATUS_NONE);

67 + fatal("Cannot setup listen port(s). Is Netdata already running?");

68 + }

70 if(unlikely(debug_flags & D_WEB_CLIENT))

71 debug_sockets();

68 -

69 - return true;

72 }

src/web/server/web_server.h

+1 -1

@@ -38,7 +38,7 @@ extern WEB_SERVER_MODE web_server_mode;

38 WEB_SERVER_MODE web_server_mode_id(const char *mode);

39 const char *web_server_mode_name(WEB_SERVER_MODE id);

41 -bool api_listen_sockets_setup(void);

41 +void web_server_listen_sockets_setup(void);

43 #define DEFAULT_TIMEOUT_TO_RECEIVE_FIRST_WEB_REQUEST 60

44 #define DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS 60