fix runtime directory; annotate daemon status file (#19706)
* fix runtime directory; annotate daemon status file * chmod the spawn server socket * do not collect system info during initialization
Costa Tsaousis committed
Feb 25, 2025 at 10:57 UTC
b001ca6943d3adf81168977b30e0db29400e07d2
8 files changed
+134
-97
src/daemon/daemon-status-file.c
+16
-4
@@ -296,7 +296,7 @@ static DAEMON_STATUS_FILE daemon_status_file_get(DAEMON_STATUS status) {
296
if(!session_status.os_id_like && last_session_status.os_id_like)
297
session_status.os_id_like = strdupz(last_session_status.os_id_like);
298
299
- if((session_status.status == DAEMON_STATUS_NONE && !session_status.architecture) || status == DAEMON_STATUS_RUNNING)
299
+ if(status == DAEMON_STATUS_RUNNING)
300
get_daemon_status_fields_from_system_info(&session_status);
301
302
session_status.exit_reason = exit_initiated;
@@ -529,7 +529,7 @@ void *post_status_file_thread(void *ptr) {
529
}
530
531
// --------------------------------------------------------------------------------------------------------------------
532
-// check last status on startup and post crash report
532
+// check last status on startup and post-crash report
533
534
void daemon_status_file_check_crash(void) {
535
last_session_status = daemon_status_file_load();
@@ -583,8 +583,20 @@ void daemon_status_file_check_crash(void) {
583
break;
584
585
case DAEMON_STATUS_INITIALIZING:
586
- cause = "crashed on start";
587
- msg = "Netdata was last killed/crashed while starting";
586
+ if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&
587
+ last_session_status.var_cache.is_read_only) {
588
+ cause = "disk read-only";
589
+ msg = "Netdata couldn't start because the disk is readonly";
590
+ }
591
+ else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&
592
+ last_session_status.var_cache.free_bytes == 0) {
593
+ cause = "disk full";
594
+ msg = "Netdata couldn't start because the disk is full";
595
+ }
596
+ else {
597
+ cause = "crashed on start";
598
+ msg = "Netdata was last killed/crashed while starting";
599
+ }
600
pri = NDLP_ERR;
601
post_crash_report = true;
602
break;
src/daemon/daemon.h
+2
@@ -10,4 +10,6 @@ void get_netdata_execution_path(void);
10
extern char *pidfile;
11
extern char *netdata_exe_path;
12
13
+void verify_required_directory(const char *env, const char *dir, bool create_it, int perms);
14
+
15
#endif /* NETDATA_DAEMON_H */
src/daemon/environment.c
+52
-30
@@ -2,35 +2,57 @@
2
3
#include "common.h"
4
5
-static const char *verify_required_directory(const char *dir)
6
-{
7
- if (chdir(dir) == -1)
8
- fatal("Cannot change directory to '%s'", dir);
5
+void verify_required_directory(const char *env, const char *dir, bool create_it, int perms) {
6
+ errno_clear();
7
10
- DIR *d = opendir(dir);
11
- if (!d)
12
- fatal("Cannot examine the contents of directory '%s'", dir);
13
- closedir(d);
8
+ if (!dir || *dir != '/')
9
+ fatal("Invalid directory path (must be an absolute path): '%s'\n", dir);
10
15
- return dir;
16
-}
11
+ if (chdir(dir) == 0) {
12
+ if(env)
13
+ nd_setenv(env, dir, 1);
14
+ return;
15
+ }
16
18
-static const char *verify_or_create_required_directory(const char *dir) {
19
- errno_clear();
17
+ if(create_it) {
18
+ if(mkdir(dir, perms) == 0) {
19
+ if(env)
20
+ nd_setenv(env, dir, 1);
21
+ return;
22
+ }
23
+ }
24
21
- if (mkdir(dir, 0755) != 0 && errno != EEXIST)
22
- fatal("Cannot create required directory '%s'", dir);
25
+ char path[PATH_MAX];
26
+ strncpyz(path, dir, sizeof(path) - 1);
27
+ struct stat st;
28
24
- return verify_required_directory(dir);
25
-}
29
+ char *p = path;
30
+ while (*p) {
31
+ if (p != path && *p == '/') {
32
+ *p = '\0';
33
27
-static const char *verify_or_create_required_private_directory(const char *dir) {
28
- errno_clear();
34
+ errno_clear();
35
+ if (stat(path, &st) == -1)
36
+ fatal("Required directory: '%s' - Missing or inaccessible component: '%s' (error: %s)\n", dir, path, strerror(errno));
37
+
38
+ if (!S_ISDIR(st.st_mode))
39
+ fatal("Required directory: '%s' - Component '%s' exists but is not a directory.\n", dir, path);
40
+
41
+ *p = '/';
42
+ }
43
+ p++;
44
+ }
45
+
46
+ if (stat(dir, &st) == -1)
47
+ fatal("Required directory: '%s' - Missing or inaccessible: '%s' (error: %s)\n", dir, dir, strerror(errno));
48
+
49
+ if (!S_ISDIR(st.st_mode))
50
+ fatal("Required directory: '%s' - '%s' exists but is not a directory.\n", dir, dir);
51
30
- if (mkdir(dir, 0770) != 0 && errno != EEXIST)
31
- fatal("Cannot create required directory '%s'", dir);
52
+ if (access(dir, R_OK | X_OK) == -1)
53
+ fatal("Required directory: '%s' - Insufficient permissions for: '%s' (error: %s)\n", dir, dir, strerror(errno));
54
33
- return verify_required_directory(dir);
55
+ fatal("Required directory: '%s' - Failed (error: %s)\n", dir, strerror(errno));
56
}
57
58
void set_environment_for_plugins_and_scripts(void) {
@@ -42,17 +64,17 @@ void set_environment_for_plugins_and_scripts(void) {
64
65
nd_setenv("NETDATA_VERSION", NETDATA_VERSION, 1);
66
nd_setenv("NETDATA_HOSTNAME", netdata_configured_hostname, 1);
45
- nd_setenv("NETDATA_CONFIG_DIR", verify_required_directory(netdata_configured_user_config_dir), 1);
46
- nd_setenv("NETDATA_USER_CONFIG_DIR", verify_required_directory(netdata_configured_user_config_dir), 1);
47
- nd_setenv("NETDATA_STOCK_CONFIG_DIR", verify_required_directory(netdata_configured_stock_config_dir), 1);
48
- nd_setenv("NETDATA_PLUGINS_DIR", verify_required_directory(netdata_configured_primary_plugins_dir), 1);
49
- nd_setenv("NETDATA_WEB_DIR", verify_required_directory(netdata_configured_web_dir), 1);
50
- nd_setenv("NETDATA_CACHE_DIR", verify_or_create_required_directory(netdata_configured_cache_dir), 1);
51
- nd_setenv("NETDATA_LIB_DIR", verify_or_create_required_directory(netdata_configured_varlib_dir), 1);
52
- nd_setenv("NETDATA_LOG_DIR", verify_or_create_required_directory(netdata_configured_log_dir), 1);
67
nd_setenv("NETDATA_HOST_PREFIX", netdata_configured_host_prefix, 1);
68
55
- nd_setenv("CLAIMING_DIR", verify_or_create_required_private_directory(netdata_configured_cloud_dir), 1);
69
+ verify_required_directory("NETDATA_CONFIG_DIR", netdata_configured_user_config_dir, false, 0);
70
+ verify_required_directory("NETDATA_USER_CONFIG_DIR", netdata_configured_user_config_dir, false, 0);
71
+ verify_required_directory("NETDATA_STOCK_CONFIG_DIR", netdata_configured_stock_config_dir, false, 0);
72
+ verify_required_directory("NETDATA_PLUGINS_DIR", netdata_configured_primary_plugins_dir, false, 0);
73
+ verify_required_directory("NETDATA_WEB_DIR", netdata_configured_web_dir, false, 0);
74
+ verify_required_directory("NETDATA_CACHE_DIR", netdata_configured_cache_dir, true, 0775);
75
+ verify_required_directory("NETDATA_LIB_DIR", netdata_configured_varlib_dir, true, 0775);
76
+ verify_required_directory("NETDATA_LOG_DIR", netdata_configured_log_dir, true, 0775);
77
+ verify_required_directory("CLAIMING_DIR", netdata_configured_cloud_dir, true, 0770);
78
79
{
80
BUFFER *user_plugins_dirs = buffer_create(FILENAME_MAX, NULL);
src/daemon/main.c
+52
-57
@@ -780,6 +780,8 @@ int netdata_main(int argc, char **argv) {
780
// }
781
}
782
783
+ nd_profile_setup();
784
+
785
// status and crash/update/exit detection
786
exit_initiated_reset();
787
daemon_status_file_check_crash();
@@ -789,12 +791,6 @@ int netdata_main(int argc, char **argv) {
791
// Get execution path before switching user to avoid permission issues
792
get_netdata_execution_path();
793
792
- // ----------------------------------------------------------------------------------------------------------------
793
- // analytics
794
-
795
- analytics_reset();
796
- get_system_timezone();
797
-
794
// ----------------------------------------------------------------------------------------------------------------
795
// data collection plugins
796
@@ -805,6 +801,12 @@ int netdata_main(int argc, char **argv) {
801
if(chdir(netdata_configured_user_config_dir) == -1)
802
fatal("Cannot cd to '%s'", netdata_configured_user_config_dir);
803
804
+ // ----------------------------------------------------------------------------------------------------------------
805
+ // analytics
806
+
807
+ analytics_reset();
808
+ get_system_timezone();
809
+
810
// ----------------------------------------------------------------------------------------------------------------
811
// pulse (internal netdata instrumentation)
812
@@ -820,83 +822,76 @@ int netdata_main(int argc, char **argv) {
822
// this has to run before starting any other threads that use workers
823
workers_utilization_enable();
824
823
- // ----------------------------------------------------------------------------------------------------------------
824
- // profiles
825
-
826
- nd_profile_setup();
827
-
825
// ----------------------------------------------------------------------------------------------------------------
826
// streaming, replication, functions initialization
827
828
replication_initialize();
829
rrd_functions_inflight_init();
830
834
- {
835
- // --------------------------------------------------------------------
836
- // alerts SILENCERS
831
+ // --------------------------------------------------------------------
832
+ // alerts SILENCERS
833
838
- health_set_silencers_filename();
839
- health_initialize_global_silencers();
834
+ health_set_silencers_filename();
835
+ health_initialize_global_silencers();
836
841
- // --------------------------------------------------------------------
842
- // setup process signals
837
+ // --------------------------------------------------------------------
838
+ // setup process signals
839
844
- // block signals while initializing threads.
845
- // this causes the threads to block signals.
840
+ // block signals while initializing threads.
841
+ // this causes the threads to block signals.
842
847
- delta_startup_time("initialize signals");
848
- nd_initialize_signals(); // setup the signals we want to use
843
+ delta_startup_time("initialize signals");
844
+ nd_initialize_signals(); // setup the signals we want to use
845
850
- // --------------------------------------------------------------------
851
- // check which threads are enabled and initialize them
846
+ // --------------------------------------------------------------------
847
+ // check which threads are enabled and initialize them
848
853
- delta_startup_time("initialize static threads");
849
+ delta_startup_time("initialize static threads");
850
855
- for (i = 0; static_threads[i].name != NULL ; i++) {
856
- struct netdata_static_thread *st = &static_threads[i];
851
+ for (i = 0; static_threads[i].name != NULL ; i++) {
852
+ struct netdata_static_thread *st = &static_threads[i];
853
858
- if(st->enable_routine)
859
- st->enabled = st->enable_routine();
854
+ if(st->enable_routine)
855
+ st->enabled = st->enable_routine();
856
861
- if(st->config_name)
862
- st->enabled = inicfg_get_boolean(&netdata_config, st->config_section, st->config_name, st->enabled);
857
+ if(st->config_name)
858
+ st->enabled = inicfg_get_boolean(&netdata_config, st->config_section, st->config_name, st->enabled);
859
864
- if(st->enabled && st->init_routine)
865
- st->init_routine();
860
+ if(st->enabled && st->init_routine)
861
+ st->init_routine();
862
867
- if(st->env_name)
868
- nd_setenv(st->env_name, st->enabled?"YES":"NO", 1);
863
+ if(st->env_name)
864
+ nd_setenv(st->env_name, st->enabled?"YES":"NO", 1);
865
870
- if(st->global_variable)
871
- *st->global_variable = (st->enabled) ? true : false;
872
- }
866
+ if(st->global_variable)
867
+ *st->global_variable = (st->enabled) ? true : false;
868
+ }
869
874
- // --------------------------------------------------------------------
875
- // create the listening sockets
870
+ // --------------------------------------------------------------------
871
+ // create the listening sockets
872
877
- delta_startup_time("initialize web server");
873
+ delta_startup_time("initialize web server");
874
879
- // get the certificate and start security
880
- netdata_conf_web_security_init();
875
+ // get the certificate and start security
876
+ netdata_conf_web_security_init();
877
882
- nd_web_api_init();
883
- web_server_threading_selection();
878
+ nd_web_api_init();
879
+ web_server_threading_selection();
880
885
- if(web_server_mode != WEB_SERVER_MODE_NONE) {
886
- if (!api_listen_sockets_setup()) {
887
- netdata_log_error("Cannot setup listen port(s). Is Netdata already running?");
888
- exit(1);
889
- }
881
+ if(web_server_mode != WEB_SERVER_MODE_NONE) {
882
+ if (!api_listen_sockets_setup()) {
883
+ netdata_log_error("Cannot setup listen port(s). Is Netdata already running?");
884
+ exit(1);
885
}
891
- if (sqlite_library_init())
892
- fatal("Failed to initialize sqlite library");
886
+ }
887
+ if (sqlite_library_init())
888
+ fatal("Failed to initialize sqlite library");
889
894
- // --------------------------------------------------------------------
895
- // Initialize ML configuration
890
+ // --------------------------------------------------------------------
891
+ // Initialize ML configuration
892
897
- delta_startup_time("initialize ML");
898
- ml_init();
899
- }
893
+ delta_startup_time("initialize ML");
894
+ ml_init();
895
896
delta_startup_time("set resource limits");
897
src/daemon/signals.c
+3
-1
@@ -116,7 +116,7 @@ void nd_process_signals(void) {
116
daemon_status_file_save(DAEMON_STATUS_NONE);
117
errno_clear();
118
119
- // loop once, but keep looping while signals are coming in
119
+ // loop once, but keep looping while signals are coming in,
120
// this is needed because a few operations may take some time
121
// so we need to check for new signals before pausing again
122
int found = 1;
@@ -155,7 +155,9 @@ void nd_process_signals(void) {
155
break;
156
157
case NETDATA_SIGNAL_FATAL:
158
+ nd_log_limits_unlimited();
159
exit_initiated_set(signals_waiting[i].reason);
160
+ daemon_status_file_save(DAEMON_STATUS_NONE);
161
fatal("SIGNAL: Received %s. netdata now exits.", name);
162
break;
163
src/libnetdata/os/run_dir.c
+1
-1
@@ -106,7 +106,7 @@ static char *detect_run_dir(bool rw) {
106
success:
107
// Set the environment variable for child processes
108
if(rw)
109
- setenv("NETDATA_RUN_DIR", path, 1);
109
+ nd_setenv("NETDATA_RUN_DIR", path, 1);
110
111
return strdupz(path);
112
}
src/libnetdata/spawn_server/spawn_server_nofork.c
+7
-2
@@ -986,6 +986,9 @@ static bool spawn_server_create_listening_socket(SPAWN_SERVER *server) {
986
return false;
987
}
988
989
+ if(chmod(server->path, 0770) != 0)
990
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: failed to chmod '%s' to 0770", server->path);
991
+
992
return true;
993
}
994
@@ -1028,8 +1031,10 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name
1031
server->id = __atomic_add_fetch(&spawn_server_id, 1, __ATOMIC_RELAXED);
1032
os_uuid_generate_random(server->magic.uuid);
1033
1031
- char *runtime_directory = getenv("NETDATA_CACHE_DIR");
1032
- if(runtime_directory && !*runtime_directory) runtime_directory = NULL;
1034
+ const char *runtime_directory = getenv("NETDATA_RUN_DIR");
1035
+ if(!runtime_directory || !*runtime_directory)
1036
+ runtime_directory = os_run_dir(true);
1037
+
1038
if (runtime_directory) {
1039
struct stat statbuf;
1040
src/registry/registry_init.c
+1
-2
@@ -82,8 +82,7 @@ void registry_init(void) {
82
// path names
83
snprintfz(filename, FILENAME_MAX, "%s/registry", netdata_configured_varlib_dir);
84
registry.pathname = inicfg_get(&netdata_config, CONFIG_SECTION_DIRECTORIES, "registry", filename);
85
- if(mkdir(registry.pathname, 0770) == -1 && errno != EEXIST)
86
- fatal("Cannot create directory '%s'.", registry.pathname);
85
+ verify_required_directory(NULL, registry.pathname, true, 0770);
86
87
// filenames
88
snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname);