@cryptotaxi247 / netdata-1 / commits / 728e36596

daemon status improvements 3 (#19707)

* spawn an init spawn server while netdata runs; then stop it and run the final one * stop the old one before dropping permissions * remove the leading dot from spawn server filenames * save the status file on every step during startup * minor update * add clarity about the double use of the function

Costa Tsaousis committed Feb 25, 2025 at 12:15 UTC 728e36596bbecffe9fc41f758ee5c088dc10c901

10 files changed +93 -87

src/claim/claim-with-api.c

+1 -1

@@ -372,7 +372,7 @@ bool claim_agent(const char *url, const char *token, const char *rooms, const ch

372 bool done = false, can_retry = true;

373 size_t retries = 0;

374 do {

375 - done = send_curl_request(registry_get_this_machine_guid(), registry_get_this_machine_hostname(), token, rooms, url, proxy, insecure, &can_retry);

375 + done = send_curl_request(registry_get_this_machine_guid(true), registry_get_this_machine_hostname(), token, rooms, url, proxy, insecure, &can_retry);

376 if (done) break;

377 sleep_usec(300 * USEC_PER_MS + 100 * retries * USEC_PER_MS);

378 retries++;

src/claim/cloud-conf.c

+1 -1

@@ -79,7 +79,7 @@ void cloud_conf_init_after_registry(void) {

80 // for machine guid and hostname we have to use inicfg_set() for that they will be saved uncommented

81 if(!machine_guid || !*machine_guid)

82 - inicfg_set(&cloud_config, CONFIG_SECTION_GLOBAL, "machine_guid", registry_get_this_machine_guid());

82 + inicfg_set(&cloud_config, CONFIG_SECTION_GLOBAL, "machine_guid", registry_get_this_machine_guid(true));

84 if(!hostname || !*hostname)

85 inicfg_set(&cloud_config, CONFIG_SECTION_GLOBAL, "hostname", registry_get_this_machine_hostname());

src/daemon/daemon-status-file.c

+13 -5

@@ -263,7 +263,7 @@ static DAEMON_STATUS_FILE daemon_status_file_get(DAEMON_STATUS status) {

263 else if(!UUIDiszero(last_session_status.host_id))

264 session_status.host_id = last_session_status.host_id;

265 else {

266 - const char *machine_guid = registry_get_this_machine_guid();

266 + const char *machine_guid = registry_get_this_machine_guid(false);

267 if(machine_guid && *machine_guid) {

268 if (uuid_parse_flexi(machine_guid, session_status.host_id.uuid) != 0)

269 session_status.host_id = UUID_ZERO;

@@ -296,8 +296,7 @@ static DAEMON_STATUS_FILE daemon_status_file_get(DAEMON_STATUS status) {

296 if(!session_status.os_id_like && last_session_status.os_id_like)

297 session_status.os_id_like = strdupz(last_session_status.os_id_like);

298

299 - if(status == DAEMON_STATUS_RUNNING)

300 - get_daemon_status_fields_from_system_info(&session_status);

299 + get_daemon_status_fields_from_system_info(&session_status);

300

301 session_status.exit_reason = exit_initiated;

302 session_status.profile = nd_profile_detect_and_configure(false);

@@ -373,7 +372,7 @@ DAEMON_STATUS_FILE daemon_status_file_load(void) {

372 char current_filename[FILENAME_MAX];

373 time_t newest_mtime = 0, current_mtime;

374

376 - // Check primary directory first

375 + // Check the primary directory first

376 if(check_status_file(netdata_configured_cache_dir, current_filename, sizeof(current_filename), &current_mtime)) {

377 strncpyz(newest_filename, current_filename, sizeof(newest_filename) - 1);

378 newest_mtime = current_mtime;

@@ -675,6 +674,13 @@ bool daemon_status_file_was_incomplete_shutdown(void) {

674 return last_session_status.status == DAEMON_STATUS_EXITING;

675 }

676

677 +void daemon_status_file_startup_step(const char *step) {

678 + freez((char *)session_status.fatal.function);

679 + session_status.fatal.function = strdupz(step);

680 + if(step != NULL)

681 + daemon_status_file_save(DAEMON_STATUS_NONE);

682 +}

683 +

684 // --------------------------------------------------------------------------------------------------------------------

685 // ng_log() hook for receiving fatal message information

686

@@ -682,7 +688,8 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio

688 static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

689 spinlock_lock(&spinlock);

690

685 - if(session_status.fatal.filename || session_status.fatal.function || session_status.fatal.message || session_status.fatal.stack_trace) {

691 + // do not check the function, because it may have a startup step in it

692 + if(session_status.fatal.filename || session_status.fatal.message || session_status.fatal.stack_trace) {

693 spinlock_unlock(&spinlock);

694 freez((void *)filename);

695 freez((void *)function);

@@ -692,6 +699,7 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio

699 }

700

701 session_status.fatal.filename = filename;

702 + freez((char *)session_status.fatal.function); // it may have a startup step

703 session_status.fatal.function = function;

704 session_status.fatal.message = message;

705 session_status.fatal.stack_trace = stack_trace;

src/daemon/daemon-status-file.h

@@ -79,6 +79,7 @@ void daemon_status_file_check_crash(void);

80 bool daemon_status_file_has_last_crashed(void);

81 bool daemon_status_file_was_incomplete_shutdown(void);

82 +void daemon_status_file_startup_step(const char *step);

84 void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *stack_trace, long line);

src/daemon/main.c

+69 -68

@@ -184,15 +184,16 @@ int help(int exitcode) {

184 be set in this procedure to be called in all the relevant code paths.

185 */

186

187 -#define delta_startup_time(msg) \

188 - { \

189 - usec_t now_ut = now_monotonic_usec(); \

190 - if(prev_msg) \

187 +#define delta_startup_time(msg) \

188 + { \

189 + usec_t now_ut = now_monotonic_usec(); \

190 + if(prev_msg) \

191 netdata_log_info("NETDATA STARTUP: in %7llu ms, %s - next: %s", (now_ut - last_ut) / USEC_PER_MS, prev_msg, msg); \

192 - else \

193 - netdata_log_info("NETDATA STARTUP: next: %s", msg); \

194 - last_ut = now_ut; \

195 - prev_msg = msg; \

192 + else \

193 + netdata_log_info("NETDATA STARTUP: next: %s", msg); \

194 + last_ut = now_ut; \

195 + prev_msg = msg; \

196 + daemon_status_file_startup_step("startup(" msg ")"); \

197 }

198

199 int buffer_unittest(void);

@@ -782,18 +783,21 @@ int netdata_main(int argc, char **argv) {

783

784 nd_profile_setup();

785

786 + // start a temporary spawn server

787 + netdata_main_spawn_server_init("init", argc, (const char **)argv);

788 +

789 // status and crash/update/exit detection

790 exit_initiated_reset();

791 daemon_status_file_check_crash();

792

793 + // ----------------------------------------------------------------------------------------------------------------

794 + delta_startup_time("initialize environment");

795 +

796 netdata_conf_ssl();

797

791 - // Get execution path before switching user to avoid permission issues

798 + // Get the execution path before switching user to avoid permission issues

799 get_netdata_execution_path();

800

794 - // ----------------------------------------------------------------------------------------------------------------

795 - // data collection plugins

796 -

801 // prepare configuration environment variables for the plugins

802 set_environment_for_plugins_and_scripts();

803

@@ -802,13 +806,13 @@ int netdata_main(int argc, char **argv) {

806 fatal("Cannot cd to '%s'", netdata_configured_user_config_dir);

807

808 // ----------------------------------------------------------------------------------------------------------------

805 - // analytics

809 + delta_startup_time("initialize analytics");

810

811 analytics_reset();

812 get_system_timezone();

813

814 // ----------------------------------------------------------------------------------------------------------------

811 - // pulse (internal netdata instrumentation)

815 + delta_startup_time("initialize pulse");

816

817 #ifdef NETDATA_INTERNAL_CHECKS

818 pulse_enabled = true;

@@ -823,29 +827,23 @@ int netdata_main(int argc, char **argv) {

827 workers_utilization_enable();

828

829 // ----------------------------------------------------------------------------------------------------------------

826 - // streaming, replication, functions initialization

830 + delta_startup_time("initialize streaming and replication");

831

832 replication_initialize();

833 rrd_functions_inflight_init();

834

831 - // --------------------------------------------------------------------

832 - // alerts SILENCERS

835 + // ----------------------------------------------------------------------------------------------------------------

836 + delta_startup_time("initialize silencers");

837

838 health_set_silencers_filename();

839 health_initialize_global_silencers();

840

837 - // --------------------------------------------------------------------

838 - // setup process signals

839 -

840 - // block signals while initializing threads.

841 - // this causes the threads to block signals.

842 -

841 + // ----------------------------------------------------------------------------------------------------------------

842 delta_startup_time("initialize signals");

844 - nd_initialize_signals(); // setup the signals we want to use

843

846 - // --------------------------------------------------------------------

847 - // check which threads are enabled and initialize them

844 + nd_initialize_signals();

845

846 + // ----------------------------------------------------------------------------------------------------------------

847 delta_startup_time("initialize static threads");

848

849 for (i = 0; static_threads[i].name != NULL ; i++) {

@@ -867,9 +865,7 @@ int netdata_main(int argc, char **argv) {

865 *st->global_variable = (st->enabled) ? true : false;

866 }

867

870 - // --------------------------------------------------------------------

871 - // create the listening sockets

872 -

868 + // ----------------------------------------------------------------------------------------------------------------

869 delta_startup_time("initialize web server");

870

871 // get the certificate and start security

@@ -884,15 +880,19 @@ int netdata_main(int argc, char **argv) {

880 exit(1);

881 }

882 }

883 +

884 + // ----------------------------------------------------------------------------------------------------------------

885 + delta_startup_time("initialize sqlite");

886 +

887 if (sqlite_library_init())

888 fatal("Failed to initialize sqlite library");

889

890 - // --------------------------------------------------------------------

891 - // Initialize ML configuration

892 -

890 + // ----------------------------------------------------------------------------------------------------------------

891 delta_startup_time("initialize ML");

892 +

893 ml_init();

894

895 + // ----------------------------------------------------------------------------------------------------------------

896 delta_startup_time("set resource limits");

897

898 #ifdef NETDATA_INTERNAL_CHECKS

@@ -908,10 +908,14 @@ int netdata_main(int argc, char **argv) {

908

909 set_nofile_limit(&rlimit_nofile);

910

911 + // ----------------------------------------------------------------------------------------------------------------

912 delta_startup_time("become daemon");

913

914 + // stop the old server and later start a new one under the new permissions

915 + netdata_main_spawn_server_cleanup();

916 +

917 #if defined(OS_LINUX) || defined(OS_MACOS) || defined(OS_FREEBSD)

914 - // fork, switch user, create pid file, set process priority

918 + // fork, switch user, create the pid file, set process priority

919 if(become_daemon(dont_fork, user) == -1)

920 fatal("Cannot daemonize myself.");

921 #else

@@ -920,52 +924,59 @@ int netdata_main(int argc, char **argv) {

924

925 netdata_main_spawn_server_init("plugins", argc, (const char **)argv);

926

923 - // init sentry

927 #ifdef ENABLE_SENTRY

925 - nd_sentry_init();

928 + // ----------------------------------------------------------------------------------------------------------------

929 + delta_startup_time("initialize sentry");

930 +

931 + nd_sentry_init();

932 #endif

933

934 + // ----------------------------------------------------------------------------------------------------------------

935 + delta_startup_time("initialize home");

936 +

937 // The "HOME" env var points to the root's home dir because Netdata starts as root. Can't use "HOME".

938 struct passwd *pw = getpwuid(getuid());

939 if (inicfg_exists(&netdata_config, CONFIG_SECTION_DIRECTORIES, "home") || !pw || !pw->pw_dir) {

940 netdata_configured_home_dir = inicfg_get(&netdata_config, CONFIG_SECTION_DIRECTORIES, "home", netdata_configured_home_dir);

932 - } else {

933 - netdata_configured_home_dir = inicfg_get(&netdata_config, CONFIG_SECTION_DIRECTORIES, "home", pw->pw_dir);

941 }

942 + else

943 + netdata_configured_home_dir = inicfg_get(&netdata_config, CONFIG_SECTION_DIRECTORIES, "home", pw->pw_dir);

944

945 nd_setenv("HOME", netdata_configured_home_dir, 1);

946

938 - dyncfg_init(true);

947 + // ----------------------------------------------------------------------------------------------------------------

948 + delta_startup_time("initialize dyncfg");

949

940 - netdata_log_info("netdata started on pid %d.", getpid());

950 + dyncfg_init(true);

951

952 + // ----------------------------------------------------------------------------------------------------------------

953 delta_startup_time("initialize threads after fork");

954

955 netdata_threads_init_after_fork((size_t)inicfg_get_size_bytes(&netdata_config, CONFIG_SECTION_GLOBAL, "pthread stack size", default_stacksize));

956

946 - // initialize internal registry

957 + // ----------------------------------------------------------------------------------------------------------------

958 delta_startup_time("initialize registry");

959 +

960 registry_load();

961 cloud_conf_init_after_registry();

962 netdata_random_session_id_generate();

963

952 - // ------------------------------------------------------------------------

953 - // initialize rrd, registry, health, streaming, etc.

954 -

955 - delta_startup_time("collecting system info");

956 -

957 - struct rrdhost_system_info *system_info = rrdhost_system_info_create();

958 - rrdhost_system_info_detect(system_info);

959 -

960 - const char *guid = registry_get_this_machine_guid();

964 + const char *guid = registry_get_this_machine_guid(true);

965 #ifdef ENABLE_SENTRY

966 nd_sentry_set_user(guid);

967 #else

968 UNUSED(guid);

969 #endif

970

971 + // ----------------------------------------------------------------------------------------------------------------

972 + delta_startup_time("collecting system info");

973 +

974 + struct rrdhost_system_info *system_info = rrdhost_system_info_create();

975 + rrdhost_system_info_detect(system_info);

976 +

977 get_install_type(system_info);

978

979 + // ----------------------------------------------------------------------------------------------------------------

980 delta_startup_time("initialize RRD structures");

981

982 abort_on_fatal_disable();

@@ -974,29 +985,19 @@ int netdata_main(int argc, char **argv) {

985 fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname);

986 }

987 abort_on_fatal_enable();

988 + reload_host_labels();

989

978 - // ------------------------------------------------------------------------

979 - // Claim netdata agent to a cloud endpoint

980 -

990 + // ----------------------------------------------------------------------------------------------------------------

991 delta_startup_time("collect claiming info");

992 +

993 + bearer_tokens_init();

994 load_claiming_state();

995

984 - // ------------------------------------------------------------------------

985 - // enable log flood protection

996 + // ----------------------------------------------------------------------------------------------------------------

997 + delta_startup_time("start the static threads");

998

999 nd_log_limits_reset();

988 -

989 - // Load host labels

990 - delta_startup_time("collect host labels");

991 - reload_host_labels();

992 -

993 - // ------------------------------------------------------------------------

994 - // spawn the threads

995 -

1000 get_agent_event_time_median_init();

997 - bearer_tokens_init();

998 -

999 - delta_startup_time("start the static threads");

1001

1002 netdata_conf_section_web();

1003

@@ -1013,13 +1014,12 @@ int netdata_main(int argc, char **argv) {

1014 }

1015 ml_start_threads();

1016

1016 - // ------------------------------------------------------------------------

1017 - // Initialize netdata agent command serving from cli and signals

1018 -

1017 + // ----------------------------------------------------------------------------------------------------------------

1018 delta_startup_time("initialize commands API");

1019

1020 commands_init();

1021

1022 + // ----------------------------------------------------------------------------------------------------------------

1023 delta_startup_time("ready");

1024

1025 usec_t ready_ut = now_monotonic_usec();

@@ -1057,6 +1057,7 @@ int netdata_main(int argc, char **argv) {

1057

1058 webrtc_initialize();

1059

1060 + daemon_status_file_startup_step(NULL);

1061 daemon_status_file_save(DAEMON_STATUS_RUNNING);

1062 return 10;

1063 }

src/database/rrd.c

+1 -1

@@ -121,7 +121,7 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool

121 localhost = rrdhost_create(

122 hostname

123 , registry_get_this_machine_hostname()

124 - , registry_get_this_machine_guid()

124 + , registry_get_this_machine_guid(true)

125 , os_type

126 , netdata_configured_timezone

127 , netdata_configured_abbrev_timezone

src/database/rrdhost-system-info.c

-5

@@ -562,12 +562,7 @@ void rrdhost_system_info_to_streaming_function_array(BUFFER *wb, struct rrdhost_

562 void get_daemon_status_fields_from_system_info(DAEMON_STATUS_FILE *ds) {

563 if(ds->read_system_info) return;

564

565 - struct rrdhost_system_info tmp = { 0 };

565 struct rrdhost_system_info *ri = (localhost && localhost->system_info) ? localhost->system_info : NULL;

567 -

568 - if(!ri && rrdhost_system_info_detect(&tmp) == 0)

569 - ri = &tmp;

570 -

566 if(!ri) {

567 // nothing we can do, let it be

568 return;

src/libnetdata/spawn_server/spawn_server_nofork.c

+2 -2

@@ -1063,11 +1063,11 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name

1063 char path[1024];

1064 if(name && *name) {

1065 server->name = strdupz(name);

1066 - snprintf(path, sizeof(path), "%s/.netdata-spawn-%s.sock", runtime_directory, name);

1066 + snprintf(path, sizeof(path), "%s/netdata-spawn-%s.sock", runtime_directory, name);

1067 }

1068 else {

1069 server->name = strdupz("unnamed");

1070 - snprintf(path, sizeof(path), "%s/.netdata-spawn-%d-%zu.sock", runtime_directory, getpid(), server->id);

1070 + snprintf(path, sizeof(path), "%s/netdata-spawn-%d-%zu.sock", runtime_directory, getpid(), server->id);

1071 }

1072

1073 server->path = strdupz(path);

src/registry/registry.h

+1 -1

@@ -75,7 +75,7 @@ void registry_update_cloud_base_url();

75 // update the registry monitoring charts

76 void registry_statistics(void);

78 -const char *registry_get_this_machine_guid(void);

78 +const char *registry_get_this_machine_guid(bool create_it);

79 char *registry_get_mgmt_api_key(void);

80 const char *registry_get_this_machine_hostname(void);

src/registry/registry_internals.c

+4 -3

@@ -270,7 +270,7 @@ const char *registry_get_this_machine_hostname(void) {

270 return registry.hostname;

271 }

272

273 -const char *registry_get_this_machine_guid(void) {

273 +const char *registry_get_this_machine_guid(bool create_it) {

274 static char guid[GUID_LEN + 1] = "";

275

276 if(likely(guid[0]))

@@ -297,7 +297,7 @@ const char *registry_get_this_machine_guid(void) {

297 }

298

299 // generate a new one?

300 - if(!guid[0]) {

300 + if(!guid[0] && create_it) {

301 nd_uuid_t uuid;

302

303 uuid_generate_time(uuid);

@@ -315,7 +315,8 @@ const char *registry_get_this_machine_guid(void) {

315 close(fd);

316 }

317

318 - nd_setenv("NETDATA_REGISTRY_UNIQUE_ID", guid, 1);

318 + if(guid[0])

319 + nd_setenv("NETDATA_REGISTRY_UNIQUE_ID", guid, 1);

320

321 return guid;

322 }