@cryptotaxi247 / netdata-1 / commits / fc1fb3285

spawn-server-nofork: invalid magic (#18831)

* debug code to trace the issue * detect the libsystemd socket and keep it open when forking * fix the path * evaluate 2k fds * increase to 1k pad buffer * detect it in a different way * cleanup * use the same journal direct socket detection * removed debugging paddding

Costa Tsaousis committed Oct 21, 2024 at 15:22 UTC fc1fb32851e400a5f84d60c71716f7c39371d0c2
6 files changed +122 -66
src/libnetdata/log/nd_log-init.c
+11 -17
@@ -44,7 +44,7 @@ void nd_log_initialize_for_external_plugins(const char *name) {
44 program_name = name;
45
46 for(size_t i = 0; i < _NDLS_MAX ;i++) {
47 - nd_log.sources[i].method = STDERR_FILENO;
47 + nd_log.sources[i].method = NDLM_DEFAULT;
48 nd_log.sources[i].fd = -1;
49 nd_log.sources[i].fp = NULL;
50 }
@@ -123,12 +123,10 @@ void nd_log_initialize_for_external_plugins(const char *name) {
123 break;
124 }
125
126 - for(size_t i = 0; i < _NDLS_MAX ;i++) {
127 - nd_log.sources[i].method = method;
128 - nd_log.sources[i].format = format;
129 - nd_log.sources[i].fd = -1;
130 - nd_log.sources[i].fp = NULL;
131 - }
126 + nd_log.sources[NDLS_COLLECTORS].method = method;
127 + nd_log.sources[NDLS_COLLECTORS].format = format;
128 + nd_log.sources[NDLS_COLLECTORS].fd = -1;
129 + nd_log.sources[NDLS_COLLECTORS].fp = NULL;
130
131 // nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
132 }
@@ -271,6 +269,10 @@ void nd_log_reopen_log_files(bool log) {
269 netdata_log_info("Log files re-opened.");
270 }
271
272 +int nd_log_systemd_journal_fd(void) {
273 + return nd_log.journal.fd;
274 +}
275 +
276 void nd_log_reopen_log_files_for_spawn_server(const char *name) {
277 gettid_uncached();
278
@@ -284,13 +286,11 @@ void nd_log_reopen_log_files_for_spawn_server(const char *name) {
286 close(nd_log.journal_direct.fd);
287 nd_log.journal_direct.fd = -1;
288 nd_log.journal_direct.initialized = false;
287 - nd_log_journal_direct_init(NULL);
289 }
290
291 for(size_t i = 0; i < _NDLS_MAX ;i++) {
291 - if(i != NDLS_COLLECTORS && i != NDLS_DAEMON) continue;
292 -
292 spinlock_init(&nd_log.sources[i].spinlock);
293 + nd_log.sources[i].method = NDLM_DEFAULT;
294 nd_log.sources[i].fd = -1;
295 nd_log.sources[i].fp = NULL;
296 nd_log.sources[i].pending_msg = NULL;
@@ -299,16 +299,10 @@ void nd_log_reopen_log_files_for_spawn_server(const char *name) {
299 #endif
300 }
301
302 - for(size_t i = 0; i < _NDLS_MAX ;i++) {
303 - if(i == NDLS_COLLECTORS || i == NDLS_DAEMON) continue;
304 - nd_log.sources[i].method = NDLM_DISABLED;
305 - }
306 -
302 + // initialize spinlocks
303 spinlock_init(&nd_log.std_output.spinlock);
304 spinlock_init(&nd_log.std_error.spinlock);
305
310 - nd_log.journal.initialized = false;
311 - nd_log.journal_direct.initialized = false;
306 nd_log.syslog.initialized = false;
307 nd_log.eventlog.initialized = false;
308 nd_log.std_output.initialized = false;
src/libnetdata/log/nd_log-internals.c
+2
@@ -284,6 +284,8 @@ struct nd_log nd_log = {
284 .overwrite_process_source = 0,
285 .journal = {
286 .initialized = false,
287 + .first_msg = false,
288 + .fd = -1,
289 },
290 .journal_direct = {
291 .initialized = false,
src/libnetdata/log/nd_log-internals.h
+3 -1
@@ -133,12 +133,14 @@ struct nd_log {
133
134 struct {
135 bool initialized;
136 + bool first_msg;
137 + int fd; // we don't control this, we just detect it to keep it open
138 } journal;
139
140 struct {
141 bool initialized;
142 int fd;
141 - char filename[FILENAME_MAX + 1];
143 + char filename[FILENAME_MAX];
144 } journal_direct;
145
146 struct {
src/libnetdata/log/nd_log-to-systemd-journal.c
+51 -27
@@ -12,18 +12,36 @@ bool nd_log_journal_systemd_init(void) {
12 return nd_log.journal.initialized;
13 }
14
15 -bool nd_log_journal_socket_available(void) {
16 - if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
17 - char filename[FILENAME_MAX + 1];
15 +static int nd_log_journal_direct_fd_find_and_open(char *filename, size_t size) {
16 + int fd;
17
19 - snprintfz(filename, sizeof(filename), "%s%s",
20 - netdata_configured_host_prefix, "/run/systemd/journal/socket");
18 + if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
19 + journal_construct_path(filename, size, netdata_configured_host_prefix, "netdata");
20 + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1)
21 + return fd;
22
22 - if(is_path_unix_socket(filename))
23 - return true;
23 + journal_construct_path(filename, size, netdata_configured_host_prefix, NULL);
24 + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1)
25 + return fd;
26 }
27
26 - return is_path_unix_socket("/run/systemd/journal/socket");
28 + journal_construct_path(filename, size, NULL, "netdata");
29 + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1)
30 + return fd;
31 +
32 + journal_construct_path(filename, size, NULL, NULL);
33 + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1)
34 + return fd;
35 +
36 + return -1;
37 +}
38 +
39 +bool nd_log_journal_socket_available(void) {
40 + char filename[FILENAME_MAX];
41 + int fd = nd_log_journal_direct_fd_find_and_open(filename, sizeof(filename));
42 + if(fd == -1) return false;
43 + close(fd);
44 + return true;
45 }
46
47 static void nd_log_journal_direct_set_env(void) {
@@ -38,25 +56,9 @@ bool nd_log_journal_direct_init(const char *path) {
56 }
57
58 int fd;
41 - char filename[FILENAME_MAX + 1];
42 - if(!is_path_unix_socket(path)) {
43 -
44 - journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata");
45 - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
46 -
47 - journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL);
48 - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
49 -
50 - journal_construct_path(filename, sizeof(filename), NULL, "netdata");
51 - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
52 -
53 - journal_construct_path(filename, sizeof(filename), NULL, NULL);
54 - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1)
55 - return false;
56 - }
57 - }
58 - }
59 - }
59 + char filename[FILENAME_MAX];
60 + if(!is_path_unix_socket(path))
61 + fd = nd_log_journal_direct_fd_find_and_open(filename, sizeof(filename));
62 else {
63 snprintfz(filename, sizeof(filename), "%s", path);
64 fd = journal_direct_fd(filename);
@@ -74,6 +76,8 @@ bool nd_log_journal_direct_init(const char *path) {
76 return true;
77 }
78
79 +static bool sockets_before[1024];
80 +
81 bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) {
82 #ifdef HAVE_SYSTEMD
83
@@ -154,8 +158,28 @@ bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_
158 }
159 }
160
161 + bool detect_systemd_socket = __atomic_load_n(&nd_log.journal.first_msg, __ATOMIC_RELAXED) == false;
162 + if(detect_systemd_socket) {
163 + for(int i = 3 ; (size_t)i < _countof(sockets_before); i++)
164 + sockets_before[i] = fd_is_socket(i);
165 + }
166 +
167 int r = sd_journal_sendv(iov, iov_count);
168
169 + if(r == 0 && detect_systemd_socket) {
170 + __atomic_store_n(&nd_log.journal.first_msg, true, __ATOMIC_RELAXED);
171 +
172 + // this is the first successful libsystemd log
173 + // let's detect its fd number (we need it for the spawn server)
174 +
175 + for(int i = 3 ; (size_t)i < _countof(sockets_before); i++) {
176 + if (!sockets_before[i] && fd_is_socket(i)) {
177 + nd_log.journal.fd = i;
178 + break;
179 + }
180 + }
181 + }
182 +
183 // Clean up allocated memory
184 for (int i = 0; i < iov_count; i++) {
185 if (iov[i].iov_base != NULL) {
src/libnetdata/log/nd_log.h
+1
@@ -14,6 +14,7 @@ extern "C" {
14 #define ND_LOG_DEFAULT_THROTTLE_PERIOD 60
15
16 void errno_clear(void);
17 +int nd_log_systemd_journal_fd(void);
18 void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting);
19 void nd_log_set_facility(const char *facility);
20 void nd_log_set_priority_level(const char *setting);
src/libnetdata/spawn_server/spawn_server_nofork.c
+54 -21
@@ -274,7 +274,13 @@ static bool spawn_external_command(SPAWN_SERVER *server __maybe_unused, SPAWN_RE
274 return false;
275 }
276
277 - os_close_all_non_std_open_fds_except(rq->fds, 3, CLOSE_RANGE_CLOEXEC);
277 + int fds_to_keep[] = {
278 + rq->fds[0],
279 + rq->fds[1],
280 + rq->fds[2],
281 + nd_log_systemd_journal_fd(),
282 + };
283 + os_close_all_non_std_open_fds_except(fds_to_keep, _countof(fds_to_keep), CLOSE_RANGE_CLOEXEC);
284
285 errno_clear();
286 if (posix_spawn(&rq->pid, rq->argv[0], &file_actions, &attr, (char * const *)rq->argv, (char * const *)rq->envp) != 0) {
@@ -325,7 +331,14 @@ static bool spawn_server_run_callback(SPAWN_SERVER *server __maybe_unused, SPAWN
331 os_setproctitle("spawn-callback", server->argc, server->argv);
332
333 // close all open file descriptors of the parent, but keep ours
328 - os_close_all_non_std_open_fds_except(rq->fds, 4, 0);
334 + int fds_to_keep[] = {
335 + rq->fds[0],
336 + rq->fds[1],
337 + rq->fds[2],
338 + rq->fds[3],
339 + nd_log_systemd_journal_fd(),
340 + };
341 + os_close_all_non_std_open_fds_except(fds_to_keep, _countof(fds_to_keep), 0);
342 nd_log_reopen_log_files_for_spawn_server("spawn-callback");
343
344 // get the fds from the request
@@ -1076,7 +1089,12 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name
1089 os_setproctitle(buf, server->argc, server->argv);
1090
1091 replace_stdio_with_dev_null();
1079 - os_close_all_non_std_open_fds_except((int[]){ server->sock, server->pipe[1] }, 2, 0);
1092 + int fds_to_keep[] = {
1093 + server->sock,
1094 + server->pipe[1],
1095 + nd_log_systemd_journal_fd(),
1096 + };
1097 + os_close_all_non_std_open_fds_except(fds_to_keep, _countof(fds_to_keep), 0);
1098 nd_log_reopen_log_files_for_spawn_server(buf);
1099 exit(spawn_server_event_loop(server));
1100 }
@@ -1125,6 +1143,21 @@ void spawn_server_exec_destroy(SPAWN_INSTANCE *instance) {
1143 freez(instance);
1144 }
1145
1146 +static void log_invalid_magic(SPAWN_INSTANCE *instance, struct status_report *sr) {
1147 + unsigned char buf[sizeof(*sr) + 1];
1148 + memcpy(buf, sr, sizeof(*sr));
1149 + buf[sizeof(buf) - 1] = '\0';
1150 +
1151 + for(size_t i = 0; i < sizeof(buf) - 1; i++) {
1152 + if (iscntrl(buf[i]) || !isprint(buf[i]))
1153 + buf[i] = '_';
1154 + }
1155 +
1156 + nd_log(NDLS_COLLECTORS, NDLP_ERR,
1157 + "SPAWN PARENT: invalid final status report for child %d, request %zu (invalid magic %#x in response, reads like '%s')",
1158 + instance->child_pid, instance->request_id, sr->magic, buf);
1159 +}
1160 +
1161 int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *instance) {
1162 int rc = -1;
1163
@@ -1139,24 +1172,24 @@ int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
1172 "SPAWN PARENT: failed to read final status report for child %d, request %zu",
1173 instance->child_pid, instance->request_id);
1174
1142 - else if(sr.magic != STATUS_REPORT_MAGIC) {
1143 - nd_log(NDLS_COLLECTORS, NDLP_ERR,
1144 - "SPAWN PARENT: invalid final status report for child %d, request %zu (invalid magic %#x in response)",
1145 - instance->child_pid, instance->request_id, sr.magic);
1146 - }
1147 - else switch(sr.status) {
1148 - case STATUS_REPORT_EXITED:
1149 - rc = sr.exited.waitpid_status;
1150 - break;
1151 -
1152 - case STATUS_REPORT_STARTED:
1153 - case STATUS_REPORT_FAILED:
1154 - default:
1155 - errno = 0;
1156 - nd_log(NDLS_COLLECTORS, NDLP_ERR,
1157 - "SPAWN PARENT: invalid status report to exec spawn request %zu for pid %d (status = %u)",
1158 - instance->request_id, instance->child_pid, sr.status);
1159 - break;
1175 + else if(sr.magic != STATUS_REPORT_MAGIC)
1176 + log_invalid_magic(instance, &sr);
1177 + else {
1178 + switch (sr.status) {
1179 + case STATUS_REPORT_EXITED:
1180 + rc = sr.exited.waitpid_status;
1181 + break;
1182 +
1183 + case STATUS_REPORT_STARTED:
1184 + case STATUS_REPORT_FAILED:
1185 + default:
1186 + errno = 0;
1187 + nd_log(
1188 + NDLS_COLLECTORS, NDLP_ERR,
1189 + "SPAWN PARENT: invalid status report to exec spawn request %zu for pid %d (status = %u)",
1190 + instance->request_id, instance->child_pid, sr.status);
1191 + break;
1192 + }
1193 }
1194
1195 instance->child_pid = 0;