@cryptotaxi247 / netdata-1 / commits / a97af3159

status file improvements 12 (#19792)

* make sure libuv threads have the deadly signals properly unblocked * add more startup steps to find the static builds crash on raspbian * initialize signals early enough to catch early deadly signals * also track run_dir as a step

Costa Tsaousis committed Mar 7, 2025 at 19:56 UTC a97af315981170c0f45da2bb4fcf43c309a4a495

13 files changed +108 -76

src/daemon/daemon-status-file.c

+4 -8

@@ -835,15 +835,13 @@ static enum crash_report_t check_crash_reports_config(void) {

835 return rc;

836 }

837

838 -void daemon_status_file_check_crash(void) {

839 - FUNCTION_RUN_ONCE();

840 -

838 +void daemon_status_file_init(void) {

839 static_save_buffer_init();

842 -

840 mallocz_register_out_of_memory_cb(daemon_status_file_out_of_memory);

844 -

841 daemon_status_file_load(&last_session_status);

846 - daemon_status_file_startup_step("startup(read status file)");

842 +}

843 +

844 +void daemon_status_file_check_crash(void) {

845 struct log_priority pri = PRI_ALL_NORMAL;

846

847 bool new_version = strcmp(last_session_status.version, session_status.version) != 0;

@@ -1051,8 +1049,6 @@ void daemon_status_file_check_crash(void) {

1049 !dedup_already_posted(&session_status, daemon_status_file_hash(&last_session_status, msg, cause))

1050

1051 ) {

1054 - daemon_status_file_startup_step("startup(post status file)");

1055 -

1052 netdata_conf_ssl();

1053

1054 struct post_status_file_thread_data d = {

src/daemon/daemon-status-file.h

@@ -100,6 +100,7 @@ bool daemon_status_file_was_incomplete_shutdown(void);

100 void daemon_status_file_startup_step(const char *step);

101 void daemon_status_file_shutdown_step(const char *step);

102

103 +void daemon_status_file_init(void);

104 void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);

105

106 #endif //NETDATA_DAEMON_STATUS_FILE_H

src/daemon/daemon.c

+27 -25

@@ -4,24 +4,6 @@

4 #include <sched.h>

6 char *pidfile = NULL;

7 -char *netdata_exe_path = NULL;

8 -

9 -void get_netdata_execution_path(void) {

10 - struct passwd *passwd = getpwuid(getuid());

11 - char *user = (passwd && passwd->pw_name) ? passwd->pw_name : "";

12 -

13 - char b[FILENAME_MAX + 1];

14 - size_t b_size = sizeof(b) - 1;

15 - int ret = uv_exepath(b, &b_size);

16 - if (ret != 0) {

17 - fatal("Cannot start netdata without getting execution path. "

18 - "(uv_exepath(\"%s\", %zu), user: '%s', failed: %s).",

19 - b, b_size, user, uv_strerror(ret));

20 - }

21 - b[b_size] = '\0';

22 -

23 - netdata_exe_path = strdupz(b);

24 -}

8 static void fix_directory_file_permissions(const char *dirname, uid_t uid, gid_t gid, bool recursive)

9 {

@@ -422,34 +404,49 @@ static void sched_setscheduler_set(void) {

404 }

405 #endif /* HAVE_SCHED_SETSCHEDULER */

406

425 -int become_daemon(int dont_fork, const char *user)

426 -{

407 +int become_daemon(int dont_fork, const char *user) {

408 if(!dont_fork) {

409 + daemon_status_file_startup_step("startup(become daemon - fork1)");

410 int i = fork();

411 if(i == -1) {

430 - perror("cannot fork");

412 + fatal("cannot fork");

413 exit(1);

414 }

433 - if(i != 0) exit(0); // the parent

415 + if(i != 0) {

416 + // the parent

417 + exit(0);

418 + }

419 +

420 + // the child

421 gettid_uncached();

422 + nd_initialize_signals();

423

424 // become session leader

425 if (setsid() < 0) {

438 - perror("Cannot become session leader.");

426 + fatal("Cannot become session leader.");

427 exit(2);

428 }

429

430 // fork() again

431 + daemon_status_file_startup_step("startup(become daemon - fork2)");

432 i = fork();

433 if(i == -1) {

445 - perror("cannot fork");

434 + fatal("cannot fork for a second time");

435 exit(1);

436 }

448 - if(i != 0) exit(0); // the parent

437 + if(i != 0) {

438 + // the parent

439 + exit(0);

440 + }

441 +

442 + // the child

443 gettid_uncached();

444 + nd_initialize_signals();

445 }

446

447 // generate our pid file

448 + daemon_status_file_startup_step("startup(become daemon - write pid)");

449 +

450 int pidfd = -1;

451 if(pidfile && *pidfile) {

452 pidfd = open(pidfile, O_WRONLY | O_CREAT | O_CLOEXEC, 0644);

@@ -471,12 +468,15 @@ int become_daemon(int dont_fork, const char *user)

468 umask(0007);

469

470 // adjust my Out-Of-Memory score

471 + daemon_status_file_startup_step("startup(become daemon - oom)");

472 oom_score_adj();

473

474 // never become a problem

475 + daemon_status_file_startup_step("startup(become daemon - sched)");

476 sched_setscheduler_set();

477

478 if(user && *user) {

479 + daemon_status_file_startup_step("startup(become daemon - user)");

480 if(become_user(user, pidfd) != 0) {

481 netdata_log_error("Cannot become user '%s'. Continuing as we are.", user);

482 }

@@ -484,9 +484,11 @@ int become_daemon(int dont_fork, const char *user)

484 netdata_log_debug(D_SYSTEM, "Successfully became user '%s'.", user);

485 }

486 else {

487 + daemon_status_file_startup_step("startup(become daemon - dirs)");

488 prepare_required_directories(getuid(), getgid());

489 }

490

491 + daemon_status_file_startup_step("startup(become daemon - done)");

492 if(pidfd != -1)

493 close(pidfd);

494

src/daemon/daemon.h

-1

@@ -8,7 +8,6 @@ int become_daemon(int dont_fork, const char *user);

8 void get_netdata_execution_path(void);

10 extern char *pidfile;

11 -extern char *netdata_exe_path;

12 void verify_required_directory(const char *env, const char *dir, bool create_it, int perms);

src/daemon/libuv_workers.c

+14 -8

@@ -3,14 +3,8 @@

3 #include <daemon/main.h>

4 #include "libuv_workers.h"

6 -// Register workers

7 -void register_libuv_worker_jobs() {

8 - static __thread bool registered = false;

9 -

10 - if(likely(registered))

11 - return;

12 -

13 - registered = true;

6 +static void register_libuv_worker_jobs_internal(void) {

7 + signals_block_all_except_deadly();

9 worker_register("LIBUV");

@@ -99,3 +93,15 @@ void register_libuv_worker_jobs() {

93 snprintfz(buf, NETDATA_THREAD_TAG_MAX, "UV_WORKER[%d]", worker_id);

94 uv_thread_set_name_np(buf);

95 }

96 +

97 +// Register workers

98 +ALWAYS_INLINE

99 +void register_libuv_worker_jobs() {

100 + static __thread bool registered = false;

101 +

102 + if(likely(registered))

103 + return;

104 +

105 + registered = true;

106 + register_libuv_worker_jobs_internal();

107 +}

src/daemon/main.c

+32 -22

@@ -759,22 +759,30 @@ int netdata_main(int argc, char **argv) {

759

760 netdata_conf_section_logs();

761 nd_log_limits_unlimited();

762 -

763 - // initialize the log files

762 nd_log_initialize();

763 +

764 + // ----------------------------------------------------------------------------------------------------------------

765 + // this MUST be before anything else - to load the old status file before saving a new one

766 +

767 + daemon_status_file_init(); // this loads the old file

768 nd_log_register_fatal_data_cb(daemon_status_file_register_fatal);

769 nd_log_register_fatal_final_cb(fatal_status_file_save);

770 + exit_initiated_init();

771 +

772 + // ----------------------------------------------------------------------------------------------------------------

773 + delta_startup_time("signals");

774 +

775 + signals_block_all_except_deadly();

776 + nd_initialize_signals(); // catches deadly signals and stores them in the status file

777 +

778 + // ----------------------------------------------------------------------------------------------------------------

779

780 netdata_conf_section_global(); // get hostname, host prefix, profile, etc

781 registry_init(); // for machine_guid, must be after netdata_conf_section_global()

782

771 - // initialize thread - this is required before the first nd_thread_create()

772 - default_stacksize = netdata_threads_init();

773 - // musl default thread stack size is 128k, let's set it to a higher value to avoid random crashes

774 - if (default_stacksize < 1 * 1024 * 1024)

775 - default_stacksize = 1 * 1024 * 1024;

783 + // ----------------------------------------------------------------------------------------------------------------

784 + delta_startup_time("run dir");

785

777 - // make sure we are the only instance running

786 {

787 const char *run_dir = os_run_dir(true);

788 if(!run_dir) {

@@ -794,18 +802,25 @@ int netdata_main(int argc, char **argv) {

802

803 nd_profile_setup();

804

797 - // status and crash/update/exit detection

798 - signals_block_all_except_deadly();

799 - exit_initiated_reset();

800 - daemon_status_file_check_crash();

805 + // ----------------------------------------------------------------------------------------------------------------

806 + delta_startup_time("stack size");

807 +

808 + // initialize thread - this is required before the first nd_thread_create()

809 + default_stacksize = netdata_threads_init();

810 +

811 + // musl default thread stack size is 128k, let's set it to a higher value to avoid random crashes

812 + if (default_stacksize < 1 * 1024 * 1024)

813 + default_stacksize = 1 * 1024 * 1024;

814 +

815 + netdata_threads_set_stack_size(default_stacksize);

816

817 // ----------------------------------------------------------------------------------------------------------------

803 - delta_startup_time("signals");

818 + delta_startup_time("crash reports");

819

805 - nd_initialize_signals();

820 + daemon_status_file_check_crash();

821

822 // ----------------------------------------------------------------------------------------------------------------

808 - delta_startup_time("temporary spawn server");

823 + delta_startup_time("temp spawn server");

824

825 netdata_main_spawn_server_init("init", argc, (const char **)argv);

826

@@ -814,12 +829,6 @@ int netdata_main(int argc, char **argv) {

829

830 netdata_conf_ssl();

831

817 - // ----------------------------------------------------------------------------------------------------------------

818 - delta_startup_time("execution path");

819 -

820 - // Get the execution path before switching user to avoid permission issues

821 - get_netdata_execution_path();

822 -

832 // ----------------------------------------------------------------------------------------------------------------

833 delta_startup_time("environment for plugins");

834

@@ -989,7 +998,8 @@ int netdata_main(int argc, char **argv) {

998 // ----------------------------------------------------------------------------------------------------------------

999 delta_startup_time("threads after fork");

1000

992 - netdata_threads_init_after_fork((size_t)inicfg_get_size_bytes(&netdata_config, CONFIG_SECTION_GLOBAL, "pthread stack size", default_stacksize));

1001 + netdata_threads_set_stack_size(

1002 + (size_t)inicfg_get_size_bytes(&netdata_config, CONFIG_SECTION_GLOBAL, "pthread stack size", default_stacksize));

1003

1004 // ----------------------------------------------------------------------------------------------------------------

1005 delta_startup_time("registry");

src/libnetdata/exit/exit_initiated.c

+1 -1

@@ -95,7 +95,7 @@ static bool is_system_shutdown(void) {

95 static const char *self_path = NULL;

96 static OS_FILE_METADATA self = { 0 };

98 -void exit_initiated_reset(void) {

98 +void exit_initiated_init(void) {

99 exit_initiated = EXIT_REASON_NONE;

100

101 freez((char *)self_path);

src/libnetdata/exit/exit_initiated.h

+1 -1

@@ -52,7 +52,7 @@ BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(EXIT_REASON);

53 extern volatile EXIT_REASON exit_initiated;

55 -void exit_initiated_reset(void);

55 +void exit_initiated_init(void);

56 void exit_initiated_set(EXIT_REASON reason);

57 void exit_initiated_add(EXIT_REASON reason);

src/libnetdata/os/process_path.c

+18 -4

@@ -5,7 +5,7 @@

5 #if defined(OS_LINUX)

6 #include <unistd.h>

8 -char *os_get_process_path(void) {

8 +static char *os_get_process_path_internal(void) {

9 char path[PATH_MAX + 1] = "";

10 ssize_t len = readlink("/proc/self/exe", path, PATH_MAX);

@@ -22,7 +22,7 @@ char *os_get_process_path(void) {

22 #if defined(OS_FREEBSD)

23 #include <sys/sysctl.h>

25 -char *os_get_process_path(void) {

25 +static char *os_get_process_path_internal(void) {

26 char path[PATH_MAX + 1] = "";

27 int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };

28 size_t len = sizeof(path);

@@ -39,7 +39,7 @@ char *os_get_process_path(void) {

39 #if defined(OS_MACOS)

40 #include <mach-o/dyld.h>

42 -char *os_get_process_path(void) {

42 +static char *os_get_process_path_internal(void) {

43 char path[PATH_MAX + 1] = "";

44 uint32_t size = sizeof(path);

@@ -62,7 +62,7 @@ char *os_get_process_path(void) {

62 #if defined(OS_WINDOWS)

63 #include <windows.h>

65 -char *os_get_process_path(void) {

65 +static char *os_get_process_path_internal(void) {

66 wchar_t wpath[32768] = L""; // Maximum path length in Windows

67 DWORD length = GetModuleFileNameW(NULL, wpath, sizeof(wpath)/sizeof(wpath[0]));

@@ -88,3 +88,17 @@ char *os_get_process_path(void) {

88 return path;

89 }

90 #endif

91 +

92 +char *os_get_process_path(void) {

93 + char b[FILENAME_MAX + 1];

94 + size_t b_size = sizeof(b) - 1;

95 + int ret = uv_exepath(b, &b_size);

96 + if(ret == 0)

97 + b[b_size] = '\0';

98 +

99 + if (ret != 0 || access(b, R_OK) != 0)

100 + return os_get_process_path_internal();

101 +

102 + b[b_size] = '\0';

103 + return strdupz(b);

104 +}

src/libnetdata/signals/signals.c

+6 -3

@@ -34,9 +34,12 @@ void signals_unblock(int signals[], size_t count) {

34 nd_log(NDLS_COLLECTORS, NDLP_ERR, "SIGNALS: cannot unmask signals");

35 }

37 -void signals_block_all_except_deadly(void) {

38 - signals_block_all();

39 -

37 +void signals_unblock_deadly(void) {

38 int deadly_signals[] = {SIGBUS, SIGSEGV, SIGFPE, SIGILL, SIGABRT};

39 signals_unblock(deadly_signals, _countof(deadly_signals));

40 }

41 +

42 +void signals_block_all_except_deadly(void) {

43 + signals_block_all();

44 + signals_unblock_deadly();

45 +}

src/libnetdata/signals/signals.h

@@ -10,5 +10,6 @@ void signals_block_all(void);

11 void signals_unblock_one(int signo);

12 void signals_unblock(int signals[], size_t count);

13 +void signals_unblock_deadly(void);

15 #endif //NETDATA_SIGNALS_H

src/libnetdata/threads/threads.c

+2 -2

@@ -211,7 +211,7 @@ size_t netdata_threads_init(void) {

211 // ----------------------------------------------------------------------------

212 // late initialization

213

214 -void netdata_threads_init_after_fork(size_t stacksize) {

214 +void netdata_threads_set_stack_size(size_t stacksize) {

215 int i;

216

217 // set pthread stack size

@@ -234,7 +234,7 @@ void netdata_threads_init_for_external_plugins(size_t stacksize) {

234 if(default_stacksize < 1 * 1024 * 1024)

235 default_stacksize = 1 * 1024 * 1024;

236

237 - netdata_threads_init_after_fork(stacksize ? stacksize : default_stacksize);

237 + netdata_threads_set_stack_size(stacksize ? stacksize : default_stacksize);

238 }

239

240 // ----------------------------------------------------------------------------

src/libnetdata/threads/threads.h

+1 -1

@@ -69,7 +69,7 @@ int nd_thread_has_tag(void);

69 #define THREAD_TAG_STREAM_SENDER "SNDR"

71 size_t netdata_threads_init(void);

72 -void netdata_threads_init_after_fork(size_t stacksize);

72 +void netdata_threads_set_stack_size(size_t stacksize);

73 void netdata_threads_init_for_external_plugins(size_t stacksize);

75 ND_THREAD *nd_thread_create(const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg);