@cryptotaxi247 / netdata-1 / commits / dbd1b26bd

local-listeners improvements (#18798)

* - procfile adaptive allocations now support initial sizing of all buffers. - procfile lines memory footprint almost halfed (changed size_t to uint32_t). - procfile exposes basic statistics about its operations. - local-sockets can now find namespace sockets from the /proc filesystem of the host (no need for setns()). - local-sockets now have 2 modes for supporting namespace sockets collection, controlled by LOCAL_SOCKETS_USE_SETNS. - local-sockets now uses procfile to read proc files. - local-sockets with no-mnl did not receive sockets information from namespaces - fixed. - local-listeners now provide detailed statistics for various aspects of its operation. - local-listeners now provide different labels for libmnl timings and proc timings. - posix spawn server now reopens logs as an external plugin (logging to DAEMON log from spawned callbacks is restored). - posix spawn server executing a callback, now does not leak file descriptors to the child. * fixed bug in mnl code - the socket cannot be reused; added option procfile to use profile instead of the getline * eliminate excess function

Costa Tsaousis committed Oct 17, 2024 at 16:59 UTC dbd1b26bd07807ed87d855985ef7a38cdb53ec5d
11 files changed +679 -276
src/collectors/apps.plugin/apps_plugin.c
+1 -1
@@ -704,7 +704,7 @@ int main(int argc, char **argv) {
704 }
705 #endif /* NETDATA_INTERNAL_CHECKS */
706
707 - procfile_adaptive_initial_allocation = 1;
707 + procfile_set_adaptive_allocation(true, 0, 0, 0);
708 os_get_system_HZ();
709 os_get_system_cpus_uncached();
710 apps_managers_and_aggregators_init(); // before parsing args!
src/collectors/network-viewer.plugin/network-viewer.c
+12 -6
@@ -80,7 +80,7 @@ struct sockets_stats {
80 } max;
81 };
82
83 -static void local_socket_to_json_array(struct sockets_stats *st, LOCAL_SOCKET *n, uint64_t proc_self_net_ns_inode, bool aggregated) {
83 +static void local_socket_to_json_array(struct sockets_stats *st, const LOCAL_SOCKET *n, uint64_t proc_self_net_ns_inode, bool aggregated) {
84 if(n->direction == SOCKET_DIRECTION_NONE)
85 return;
86
@@ -156,7 +156,7 @@ static void local_socket_to_json_array(struct sockets_stats *st, LOCAL_SOCKET *n
156 string_freez(u);
157 }
158
159 - struct socket_endpoint *server_endpoint;
159 + const struct socket_endpoint *server_endpoint;
160 const char *server_address;
161 const char *client_address_space;
162 const char *server_address_space;
@@ -240,7 +240,9 @@ static void local_socket_to_json_array(struct sockets_stats *st, LOCAL_SOCKET *n
240 buffer_json_array_close(wb);
241 }
242
243 -static void populate_aggregated_key(LOCAL_SOCKET *n) {
243 +static void populate_aggregated_key(const LOCAL_SOCKET *nn) {
244 + LOCAL_SOCKET *n = (LOCAL_SOCKET *)nn;
245 +
246 n->network_viewer.count = 1;
247
248 n->network_viewer.aggregated_key.pid = n->pid;
@@ -269,7 +271,7 @@ static void populate_aggregated_key(LOCAL_SOCKET *n) {
271 n->network_viewer.aggregated_key.remote_address_space = local_sockets_address_space(&n->remote);
272 }
273
272 -static void local_sockets_cb_to_json(LS_STATE *ls, LOCAL_SOCKET *n, void *data) {
274 +static void local_sockets_cb_to_json(LS_STATE *ls, const LOCAL_SOCKET *n, void *data) {
275 struct sockets_stats *st = data;
276 populate_aggregated_key(n);
277 local_socket_to_json_array(st, n, ls->proc_self_net_ns_inode, false);
@@ -280,12 +282,12 @@ static void local_sockets_cb_to_json(LS_STATE *ls, LOCAL_SOCKET *n, void *data)
282 #define SUM_THEM_ALL(a, b) (a) += (b)
283 #define OR_THEM_ALL(a, b) (a) |= (b)
284
283 -static void local_sockets_cb_to_aggregation(LS_STATE *ls __maybe_unused, LOCAL_SOCKET *n, void *data) {
285 +static void local_sockets_cb_to_aggregation(LS_STATE *ls __maybe_unused, const LOCAL_SOCKET *n, void *data) {
286 SIMPLE_HASHTABLE_AGGREGATED_SOCKETS *ht = data;
287
288 populate_aggregated_key(n);
289 XXH64_hash_t hash = XXH3_64bits(&n->network_viewer.aggregated_key, sizeof(n->network_viewer.aggregated_key));
288 - SIMPLE_HASHTABLE_SLOT_AGGREGATED_SOCKETS *sl = simple_hashtable_get_slot_AGGREGATED_SOCKETS(ht, hash, n, true);
290 + SIMPLE_HASHTABLE_SLOT_AGGREGATED_SOCKETS *sl = simple_hashtable_get_slot_AGGREGATED_SOCKETS(ht, hash, (LOCAL_SOCKET *)n, true);
291 LOCAL_SOCKET *t = SIMPLE_HASHTABLE_SLOT_DATA(sl);
292 if(t) {
293 t->network_viewer.count++;
@@ -511,7 +513,9 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu
513 .max_errors = 10,
514 .max_concurrent_namespaces = 5,
515 },
516 +#if defined(LOCAL_SOCKETS_USE_SETNS)
517 .spawn_server = spawn_srv,
518 +#endif
519 .stats = { 0 },
520 .sockets_hashtable = { 0 },
521 .local_ips_hashtable = { 0 },
@@ -963,11 +967,13 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
967 netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
968 if(verify_netdata_host_prefix(true) == -1) exit(1);
969
970 +#if defined(LOCAL_SOCKETS_USE_SETNS)
971 spawn_srv = spawn_server_create(SPAWN_SERVER_OPTION_CALLBACK, "setns", local_sockets_spawn_server_callback, argc, (const char **)argv);
972 if(spawn_srv == NULL) {
973 fprintf(stderr, "Cannot create spawn server.\n");
974 exit(1);
975 }
976 +#endif
977
978 uc = system_usernames_cache_init();
979 sc = system_servicenames_cache_init();
src/collectors/slabinfo.plugin/slabinfo.c
+1 -1
@@ -167,7 +167,7 @@ struct slabinfo *read_file_slabinfo() {
167 slabdebug(" Read %lu lines from procfile", (unsigned long)lines);
168 for(l = 2; l < lines; l++) {
169 if (unlikely(procfile_linewords(ff, l) < 14)) {
170 - slabdebug(" Line %zu has only %zu words, skipping", l, procfile_linewords(ff,l));
170 + slabdebug(" Line %zu has only %zu words, skipping", l, (size_t)procfile_linewords(ff,l));
171 continue;
172 }
173
src/collectors/utils/local_listeners.c
+51 -3
@@ -35,7 +35,9 @@ static const char *protocol_name(LOCAL_SOCKET *n) {
35 return "UNKNOWN";
36 }
37
38 -static void print_local_listeners(LS_STATE *ls __maybe_unused, LOCAL_SOCKET *n, void *data __maybe_unused) {
38 +static void print_local_listeners(LS_STATE *ls __maybe_unused, const LOCAL_SOCKET *nn, void *data __maybe_unused) {
39 + LOCAL_SOCKET *n = (LOCAL_SOCKET *)nn;
40 +
41 char local_address[INET6_ADDRSTRLEN];
42 char remote_address[INET6_ADDRSTRLEN];
43
@@ -55,7 +57,9 @@ static void print_local_listeners(LS_STATE *ls __maybe_unused, LOCAL_SOCKET *n,
57 printf("%s|%s|%u|%s\n", protocol_name(n), local_address, n->local.port, string2str(n->cmdline));
58 }
59
58 -static void print_local_listeners_debug(LS_STATE *ls __maybe_unused, LOCAL_SOCKET *n, void *data __maybe_unused) {
60 +static void print_local_listeners_debug(LS_STATE *ls __maybe_unused, const LOCAL_SOCKET *nn, void *data __maybe_unused) {
61 + LOCAL_SOCKET *n = (LOCAL_SOCKET *)nn;
62 +
63 char local_address[INET6_ADDRSTRLEN];
64 char remote_address[INET6_ADDRSTRLEN];
65
@@ -155,7 +159,7 @@ int main(int argc, char **argv) {
159 "\n"
160 " Current options:\n"
161 "\n"
158 - " %s %s %s %s %s %s %s %s %s\n"
162 + " %s %s %s %s %s %s %s %s %s %s %s %s\n"
163 "\n"
164 " Option 'debug' enables all sources and all directions and provides\n"
165 " a full dump of current sockets.\n"
@@ -163,6 +167,8 @@ int main(int argc, char **argv) {
167 " Option 'report' reports timings per step while collecting and processing\n"
168 " system information.\n"
169 "\n"
170 + " Option 'procfile' uses procfile to read proc files, instead of getline().\n"
171 + "\n"
172 " DIRECTION DETECTION\n"
173 " The program detects the direction of the sockets using these rules:\n"
174 "\n"
@@ -208,6 +214,9 @@ int main(int argc, char **argv) {
214 , ls.config.inbound ? "inbound" : "no-inbound"
215 , ls.config.outbound ? "outbound" : "no-outbound"
216 , ls.config.namespaces ? "namespaces" : "no-namespaces"
217 + , ls.config.no_mnl ? "no-mnl" : "mnl"
218 + , ls.config.procfile ? "procfile" : "no-procfile"
219 + , ls.config.report ? "report" : "no-report"
220 );
221 exit(1);
222 }
@@ -233,6 +242,7 @@ int main(int argc, char **argv) {
242 ls.config.namespaces = true;
243 ls.config.tcp_info = true;
244 ls.config.uid = true;
245 + ls.config.procfile = false;
246 ls.config.max_errors = SIZE_MAX;
247 ls.config.cb = print_local_listeners_debug;
248
@@ -294,6 +304,10 @@ int main(int argc, char **argv) {
304 ls.config.no_mnl = !positive;
305 // fprintf(stderr, "%s mnl\n", positive ? "enabling" : "disabling");
306 }
307 + else if (strcmp("procfile", s) == 0) {
308 + ls.config.procfile = positive;
309 + // fprintf(stderr, "%s procfile\n", positive ? "enabling" : "disabling");
310 + }
311 else if (strcmp("report", s) == 0) {
312 ls.config.report = positive;
313 // fprintf(stderr, "%s report\n", positive ? "enabling" : "disabling");
@@ -304,16 +318,21 @@ int main(int argc, char **argv) {
318 }
319 }
320
321 +#if defined(LOCAL_SOCKETS_USE_SETNS)
322 SPAWN_SERVER *spawn_server = spawn_server_create(SPAWN_SERVER_OPTION_CALLBACK, NULL, local_sockets_spawn_server_callback, argc, (const char **)argv);
323 if(spawn_server == NULL) {
324 fprintf(stderr, "Cannot create spawn server.\n");
325 exit(1);
326 }
327 +
328 ls.spawn_server = spawn_server;
329 +#endif
330
331 local_sockets_process(&ls);
332
333 +#if defined(LOCAL_SOCKETS_USE_SETNS)
334 spawn_server_destroy(spawn_server);
335 +#endif
336
337 getrusage(RUSAGE_SELF, &ended);
338
@@ -345,6 +364,35 @@ int main(int argc, char **argv) {
364
365 duration_snprintf(buf, sizeof(buf), (int64_t)total_ut, "us", true);
366 fprintf(stderr, "%20s: %6.2f%% %s\n", "TOTAL", 100.0, buf);
367 +
368 + fprintf(stderr, "\n");
369 + fprintf(stderr, "Namespaces [ found: %zu, absent: %zu, invalid: %zu ]\n"
370 +#if defined(LOCAL_SOCKETS_USE_SETNS)
371 + " \\_ forks [ tried: %zu, failed: %zu, unresponsive: %zu ]\n"
372 + " \\_ sockets [ new: %zu, existing: %zu ]\n"
373 +#endif
374 + , ls.stats.namespaces_found, ls.stats.namespaces_absent, ls.stats.namespaces_invalid
375 +#if defined(LOCAL_SOCKETS_USE_SETNS)
376 + , ls.stats.namespaces_forks_attempted, ls.stats.namespaces_forks_failed, ls.stats.namespaces_forks_unresponsive
377 + , ls.stats.namespaces_sockets_new, ls.stats.namespaces_sockets_existing
378 +#endif
379 + );
380 +
381 + fprintf(stderr, "\n");
382 + fprintf(stderr, "Sockets [ found: %zu ]\n",
383 + ls.stats.sockets_added);
384 +
385 + fprintf(stderr, "\n");
386 + fprintf(stderr, "Main Procfile [ opens: %zu, reads: %zu, resizes: %zu, memory: %zu ]\n"
387 + " \\_ reads [ total bytes read: %zu, average read size: %zu, max read size: %zu ]\n"
388 + " \\_ max [ max file size: %zu, max lines: %zu, max words: %zu ]\n",
389 + ls.stats.ff.opens, ls.stats.ff.reads, ls.stats.ff.resizes, ls.stats.ff.memory,
390 + ls.stats.ff.total_read_bytes, ls.stats.ff.total_read_bytes / (ls.stats.ff.reads ? ls.stats.ff.reads : 1), ls.stats.ff.max_read_size,
391 + ls.stats.ff.max_source_bytes, ls.stats.ff.max_lines, ls.stats.ff.max_words);
392 +
393 + fprintf(stderr, "\n");
394 + fprintf(stderr, "MNL(without namespaces) [ requests: %zu ]\n",
395 + ls.stats.mnl_sends);
396 }
397
398 return 0;
src/libnetdata/aral/aral.c
+1
@@ -799,6 +799,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele
799 aral_delete_leftover_files(ar->config.name, directory_name, file);
800 }
801
802 + errno_clear();
803 internal_error(true,
804 "ARAL: '%s' "
805 "element size %zu (requested %zu bytes), "
src/libnetdata/log/nd_log-init.c
+29 -11
@@ -265,17 +265,13 @@ void nd_log_reopen_log_files(bool log) {
265 if(log)
266 netdata_log_info("Reopening all log files.");
267
268 - nd_log.std_output.initialized = false;
269 - nd_log.std_error.initialized = false;
270 - nd_log.journal_direct.initialized = false;
271 - nd_log.journal.initialized = false;
268 nd_log_initialize();
269
270 if(log)
271 netdata_log_info("Log files re-opened.");
272 }
273
278 -void nd_log_reopen_log_files_for_spawn_server(void) {
274 +void nd_log_reopen_log_files_for_spawn_server(const char *name) {
275 gettid_uncached();
276
277 if(nd_log.syslog.initialized) {
@@ -291,11 +287,33 @@ void nd_log_reopen_log_files_for_spawn_server(void) {
287 nd_log_journal_direct_init(NULL);
288 }
289
294 - nd_log.sources[NDLS_UNSET].method = NDLM_DISABLED;
295 - nd_log.sources[NDLS_ACCESS].method = NDLM_DISABLED;
296 - nd_log.sources[NDLS_ACLK].method = NDLM_DISABLED;
297 - nd_log.sources[NDLS_DEBUG].method = NDLM_DISABLED;
298 - nd_log.sources[NDLS_HEALTH].method = NDLM_DISABLED;
299 - nd_log_reopen_log_files(false);
290 + for(size_t i = 0; i < _NDLS_MAX ;i++) {
291 + if(i != NDLS_COLLECTORS && i != NDLS_DAEMON) continue;
292 +
293 + spinlock_init(&nd_log.sources[i].spinlock);
294 + nd_log.sources[i].fd = -1;
295 + nd_log.sources[i].fp = NULL;
296 + nd_log.sources[i].pending_msg = NULL;
297 +#if defined(OS_WINDOWS)
298 + nd_log.sources[i].hEventLog = NULL;
299 +#endif
300 + }
301 +
302 + for(size_t i = 0; i < _NDLS_MAX ;i++) {
303 + if(i == NDLS_COLLECTORS || i == NDLS_DAEMON) continue;
304 + nd_log.sources[i].method = NDLM_DISABLED;
305 + }
306 +
307 + spinlock_init(&nd_log.std_output.spinlock);
308 + spinlock_init(&nd_log.std_error.spinlock);
309 +
310 + nd_log.journal.initialized = false;
311 + nd_log.journal_direct.initialized = false;
312 + nd_log.syslog.initialized = false;
313 + nd_log.eventlog.initialized = false;
314 + nd_log.std_output.initialized = false;
315 + nd_log.std_error.initialized = false;
316 +
317 + nd_log_initialize_for_external_plugins(name);
318 }
319
src/libnetdata/log/nd_log.h
+1 -1
@@ -23,7 +23,7 @@ void chown_open_file(int fd, uid_t uid, gid_t gid);
23 void nd_log_chown_log_files(uid_t uid, gid_t gid);
24 void nd_log_set_flood_protection(size_t logs, time_t period);
25 void nd_log_initialize_for_external_plugins(const char *name);
26 -void nd_log_reopen_log_files_for_spawn_server(void);
26 +void nd_log_reopen_log_files_for_spawn_server(const char *name);
27 bool nd_log_journal_socket_available(void);
28 ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len);
29 int nd_log_priority2id(const char *priority);
src/libnetdata/maps/local-sockets.h
+453 -169
@@ -9,7 +9,10 @@
9 #define _countof(x) (sizeof(x) / sizeof(*(x)))
10 #endif
11
12 -#ifdef HAVE_LIBMNL
12 +#define LOCAL_SOCKETS_USE_SETNS
13 +#define USE_LIBMNL_AFTER_SETNS
14 +
15 +#if defined(HAVE_LIBMNL)
16 #include <linux/rtnetlink.h>
17 #include <linux/inet_diag.h>
18 #include <linux/sock_diag.h>
@@ -67,7 +70,7 @@ struct local_port;
70 // --------------------------------------------------------------------------------------------------------------------
71
72 struct local_socket_state;
70 -typedef void (*local_sockets_cb_t)(struct local_socket_state *state, struct local_socket *n, void *data);
73 +typedef void (*local_sockets_cb_t)(struct local_socket_state *state, const struct local_socket *n, void *data);
74
75 struct local_sockets_config {
76 bool listening;
@@ -85,6 +88,7 @@ struct local_sockets_config {
88 bool namespaces;
89 bool tcp_info;
90 bool no_mnl;
91 + bool procfile;
92 bool report;
93
94 size_t max_errors;
@@ -94,9 +98,12 @@ struct local_sockets_config {
98 void *data;
99
100 const char *host_prefix;
101 +};
102
98 - // internal use
103 +struct local_sockets_state {
104 + uint32_t nl_seq;
105 uint64_t net_ns_inode;
106 + pid_t net_ns_pid;
107 };
108
109 struct timing_work {
@@ -105,32 +112,54 @@ struct timing_work {
112 const char *name;
113 };
114
115 +struct local_sockets_ns_req {
116 + struct local_sockets_config config;
117 + struct local_sockets_state ns_state;
118 +};
119 +
120 typedef struct local_socket_state {
121 struct local_sockets_config config;
122 + struct local_sockets_state ns_state;
123
124 struct {
125 size_t mnl_sends;
113 - size_t namespaces_found;
126 size_t tcp_info_received;
127 size_t pid_fds_processed;
128 size_t pid_fds_opendir_failed;
129 size_t pid_fds_readlink_failed;
130 size_t pid_fds_parse_failed;
131 size_t errors_encountered;
132 +
133 + size_t sockets_added;
134 +
135 + size_t namespaces_found;
136 + size_t namespaces_absent;
137 + size_t namespaces_invalid;
138 +#if defined(LOCAL_SOCKETS_USE_SETNS)
139 + size_t namespaces_forks_attempted;
140 + size_t namespaces_forks_failed;
141 + size_t namespaces_forks_unresponsive;
142 + size_t namespaces_sockets_new;
143 + size_t namespaces_sockets_existing;
144 +#endif
145 +
146 + struct procfile_stats ff;
147 } stats;
148
149 size_t timings_idx;
123 - struct timing_work timings[20];
150 + struct timing_work timings[30];
151
152 +#if defined(LOCAL_SOCKETS_USE_SETNS)
153 bool spawn_server_is_mine;
154 SPAWN_SERVER *spawn_server;
155 +#endif
156
128 -#ifdef HAVE_LIBMNL
129 - bool use_mnl;
130 - struct mnl_socket *nl;
157 +#if defined(HAVE_LIBMNL)
158 uint16_t tmp_protocol;
159 #endif
160
161 + procfile *ff;
162 +
163 ARAL *local_socket_aral;
164 ARAL *pid_socket_aral;
165 SPINLOCK spinlock; // for namespaces
@@ -238,7 +267,9 @@ typedef struct local_socket {
267 #endif
268 } LOCAL_SOCKET;
269
270 +#if defined(LOCAL_SOCKETS_USE_SETNS)
271 static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request);
272 +#endif
273
274 // --------------------------------------------------------------------------------------------------------------------
275
@@ -269,7 +300,7 @@ static bool local_sockets_is_ipv4_mapped_ipv6_address(const struct in6_addr *add
300 return memcmp(addr->s6_addr, ipv4_mapped_prefix, 12) == 0;
301 }
302
272 -static bool local_sockets_is_loopback_address(struct socket_endpoint *se) {
303 +static bool local_sockets_is_loopback_address(const struct socket_endpoint *se) {
304 if (se->family == AF_INET) {
305 // For IPv4, loopback addresses are in the 127.0.0.0/8 range
306 return (ntohl(se->ip.ipv4) >> 24) == 127; // Check if the first byte is 127
@@ -303,7 +334,7 @@ static inline bool local_sockets_is_ipv4_reserved_address(uint32_t ip) {
334 );
335 }
336
306 -static inline bool local_sockets_is_private_address(struct socket_endpoint *se) {
337 +static inline bool local_sockets_is_private_address(const struct socket_endpoint *se) {
338 if (se->family == AF_INET) {
339 return local_sockets_is_ipv4_reserved_address(se->ip.ipv4);
340 }
@@ -337,7 +368,7 @@ static inline bool local_sockets_is_private_address(struct socket_endpoint *se)
368 return false;
369 }
370
340 -static bool local_sockets_is_multicast_address(struct socket_endpoint *se) {
371 +static bool local_sockets_is_multicast_address(const struct socket_endpoint *se) {
372 if (se->family == AF_INET) {
373 // For IPv4, check if the address is 0.0.0.0
374 uint32_t ip = htonl(se->ip.ipv4);
@@ -352,7 +383,7 @@ static bool local_sockets_is_multicast_address(struct socket_endpoint *se) {
383 return false;
384 }
385
355 -static bool local_sockets_is_zero_address(struct socket_endpoint *se) {
386 +static bool local_sockets_is_zero_address(const struct socket_endpoint *se) {
387 if (se->family == AF_INET) {
388 // For IPv4, check if the address is 0.0.0.0
389 return se->ip.ipv4 == 0;
@@ -365,7 +396,7 @@ static bool local_sockets_is_zero_address(struct socket_endpoint *se) {
396 return false;
397 }
398
368 -static inline const char *local_sockets_address_space(struct socket_endpoint *se) {
399 +static inline const char *local_sockets_address_space(const struct socket_endpoint *se) {
400 if(local_sockets_is_zero_address(se))
401 return "zero";
402 else if(local_sockets_is_loopback_address(se))
@@ -380,7 +411,7 @@ static inline const char *local_sockets_address_space(struct socket_endpoint *se
411
412 // --------------------------------------------------------------------------------------------------------------------
413
383 -static inline bool is_local_socket_ipv46(LOCAL_SOCKET *n) {
414 +static inline bool is_local_socket_ipv46(const LOCAL_SOCKET *n) {
415 return n->local.family == AF_INET6 &&
416 n->direction == SOCKET_DIRECTION_LISTEN &&
417 local_sockets_is_zero_address(&n->local) &&
@@ -615,6 +646,8 @@ static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) {
646 return false;
647 }
648
649 + ls->stats.sockets_added++;
650 +
651 n = aral_mallocz(ls->local_socket_aral);
652 *n = *tmp; // copy all contents
653
@@ -683,33 +716,7 @@ static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) {
716 return true;
717 }
718
686 -#ifdef HAVE_LIBMNL
687 -
688 -static inline void local_sockets_libmnl_init(LS_STATE *ls) {
689 - if(ls->config.no_mnl) return;
690 -
691 - ls->nl = mnl_socket_open(NETLINK_INET_DIAG);
692 - if (ls->nl == NULL) {
693 - local_sockets_log(ls, "cannot open libmnl netlink socket");
694 - ls->use_mnl = false;
695 - }
696 - else if (mnl_socket_bind(ls->nl, 0, MNL_SOCKET_AUTOPID) < 0) {
697 - local_sockets_log(ls, "cannot bind libmnl netlink socket");
698 - mnl_socket_close(ls->nl);
699 - ls->nl = NULL;
700 - ls->use_mnl = false;
701 - }
702 - else
703 - ls->use_mnl = true;
704 -}
705 -
706 -static inline void local_sockets_libmnl_cleanup(LS_STATE *ls) {
707 - if(ls->nl) {
708 - mnl_socket_close(ls->nl);
709 - ls->nl = NULL;
710 - ls->use_mnl = false;
711 - }
712 -}
719 +#if defined(HAVE_LIBMNL)
720
721 static inline int local_sockets_libmnl_cb_data(const struct nlmsghdr *nlh, void *data) {
722 LS_STATE *ls = data;
@@ -784,16 +791,30 @@ static inline int local_sockets_libmnl_cb_data(const struct nlmsghdr *nlh, void
791 static inline bool local_sockets_libmnl_get_sockets(LS_STATE *ls, uint16_t family, uint16_t protocol) {
792 ls->tmp_protocol = protocol;
793
787 - char buf[MNL_SOCKET_BUFFER_SIZE];
788 - struct nlmsghdr *nlh;
789 - struct inet_diag_req_v2 req;
790 - unsigned int seq, portid = mnl_socket_get_portid(ls->nl);
794 + struct mnl_socket *nl = mnl_socket_open(NETLINK_INET_DIAG);
795 + if (nl == NULL) {
796 + local_sockets_log(ls, "mnl_socket_open() failed");
797 + return false;
798 + }
799
792 - memset(&req, 0, sizeof(req));
793 - req.sdiag_family = family;
794 - req.sdiag_protocol = protocol;
795 - req.idiag_states = -1;
796 - req.idiag_ext = 0;
800 + if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) {
801 + local_sockets_log(ls, "mnl_socket_bind() failed");
802 + mnl_socket_close(nl);
803 + return false;
804 + }
805 +
806 + char buf[MNL_SOCKET_BUFFER_SIZE];
807 + struct nlmsghdr *nlh = mnl_nlmsg_put_header(buf);
808 + nlh->nlmsg_type = SOCK_DIAG_BY_FAMILY;
809 + nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
810 + nlh->nlmsg_seq = ls->ns_state.nl_seq ? ls->ns_state.nl_seq++ : time(NULL);
811 +
812 + struct inet_diag_req_v2 req = {
813 + .sdiag_family = family,
814 + .sdiag_protocol = protocol,
815 + .idiag_states = ~0, // Request all socket states
816 + .idiag_ext = 0,
817 + };
818
819 if(family == AF_INET6)
820 req.idiag_ext |= 1 << (INET_DIAG_SKV6ONLY - 1);
@@ -801,35 +822,106 @@ static inline bool local_sockets_libmnl_get_sockets(LS_STATE *ls, uint16_t famil
822 if(protocol == IPPROTO_TCP && ls->config.tcp_info)
823 req.idiag_ext |= 1 << (INET_DIAG_INFO - 1);
824
804 - nlh = mnl_nlmsg_put_header(buf);
805 - nlh->nlmsg_type = SOCK_DIAG_BY_FAMILY;
806 - nlh->nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST;
807 - nlh->nlmsg_seq = seq = time(NULL);
825 mnl_nlmsg_put_extra_header(nlh, sizeof(req));
826 memcpy(mnl_nlmsg_get_payload(nlh), &req, sizeof(req));
827
828 ls->stats.mnl_sends++;
812 - if (mnl_socket_sendto(ls->nl, nlh, nlh->nlmsg_len) < 0) {
813 - local_sockets_log(ls, "mnl_socket_send failed");
829 + if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) {
830 + local_sockets_log(ls, "mnl_socket_sendto() failed");
831 + mnl_socket_close(nl);
832 return false;
833 }
834
835 + bool rc = true;
836 + size_t received = 0;
837 ssize_t ret;
818 - while ((ret = mnl_socket_recvfrom(ls->nl, buf, sizeof(buf))) > 0) {
819 - ret = mnl_cb_run(buf, ret, seq, portid, local_sockets_libmnl_cb_data, ls);
820 - if (ret <= MNL_CB_STOP)
838 + while ((ret = mnl_socket_recvfrom(nl, buf, sizeof(buf))) > 0) {
839 + ret = mnl_cb_run(buf, ret, 0, 0, local_sockets_libmnl_cb_data, ls);
840 + if (ret == MNL_CB_ERROR) {
841 + local_sockets_log(ls, "mnl_cb_run() failed");
842 + rc = false;
843 break;
844 + }
845 + else if (ret <= MNL_CB_STOP)
846 + break;
847 +
848 + received++;
849 }
850 + mnl_socket_close(nl);
851 +
852 if (ret == -1) {
824 - local_sockets_log(ls, "mnl_socket_recvfrom");
853 + local_sockets_log(ls, "mnl_socket_recvfrom() failed");
854 + rc = false;
855 + }
856 +
857 + return rc;
858 +}
859 +#endif // HAVE_LIBMNL
860 +
861 +static inline bool local_sockets_process_proc_line(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol, size_t line, char **words, size_t num_words) {
862 + // char *sl_txt = get_word(words, num_words, 0);
863 + char *local_ip_txt = get_word(words, num_words, 1);
864 + char *local_port_txt = get_word(words, num_words, 2);
865 + char *remote_ip_txt = get_word(words, num_words, 3);
866 + char *remote_port_txt = get_word(words, num_words, 4);
867 + char *state_txt = get_word(words, num_words, 5);
868 + char *tx_queue_txt = get_word(words, num_words, 6);
869 + char *rx_queue_txt = get_word(words, num_words, 7);
870 + char *tr_txt = get_word(words, num_words, 8);
871 + char *tm_when_txt = get_word(words, num_words, 9);
872 + char *retrans_txt = get_word(words, num_words, 10);
873 + char *uid_txt = get_word(words, num_words, 11);
874 + // char *timeout_txt = get_word(words, num_words, 12);
875 + char *inode_txt = get_word(words, num_words, 13);
876 +
877 + if(!local_ip_txt || !local_port_txt || !remote_ip_txt || !remote_port_txt || !state_txt ||
878 + !tx_queue_txt || !rx_queue_txt || !tr_txt || !tm_when_txt || !retrans_txt || !uid_txt || !inode_txt) {
879 + local_sockets_log(ls, "cannot parse ipv4 line No %zu of filename '%s'", line, filename);
880 return false;
881 }
882
883 + LOCAL_SOCKET n = {
884 + .direction = SOCKET_DIRECTION_NONE,
885 + .ipv6ony = {
886 + .checked = false,
887 + .ipv46 = false,
888 + },
889 + .local = {
890 + .family = family,
891 + .protocol = protocol,
892 + },
893 + .remote = {
894 + .family = family,
895 + .protocol = protocol,
896 + },
897 + .uid = UID_UNSET,
898 + };
899 +
900 + n.local.port = str2uint32_hex(local_port_txt, NULL);
901 + n.remote.port = str2uint32_hex(remote_port_txt, NULL);
902 + n.state = str2uint32_hex(state_txt, NULL);
903 + n.wqueue = str2uint32_hex(tx_queue_txt, NULL);
904 + n.rqueue = str2uint32_hex(rx_queue_txt, NULL);
905 + n.timer = str2uint32_hex(tr_txt, NULL);
906 + n.expires = str2uint32_hex(tm_when_txt, NULL);
907 + n.retransmits = str2uint32_hex(retrans_txt, NULL);
908 + n.uid = str2uint32_t(uid_txt, NULL);
909 + n.inode = str2uint64_t(inode_txt, NULL);
910 +
911 + if(family == AF_INET) {
912 + n.local.ip.ipv4 = str2uint32_hex(local_ip_txt, NULL);
913 + n.remote.ip.ipv4 = str2uint32_hex(remote_ip_txt, NULL);
914 + }
915 + else if(family == AF_INET6) {
916 + ipv6_to_in6_addr(local_ip_txt, &n.local.ip.ipv6);
917 + ipv6_to_in6_addr(remote_ip_txt, &n.remote.ip.ipv6);
918 + }
919 +
920 + local_sockets_add_socket(ls, &n);
921 return true;
922 }
830 -#endif // HAVE_LIBMNL
923
832 -static inline bool local_sockets_read_proc_net_x(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) {
924 +static inline bool local_sockets_read_proc_net_x_getline(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) {
925 static bool is_space[256] = {
926 [':'] = true,
927 [' '] = true,
@@ -863,67 +955,9 @@ static inline bool local_sockets_read_proc_net_x(LS_STATE *ls, const char *filen
955 continue;
956 }
957
866 - LOCAL_SOCKET n = {
867 - .direction = SOCKET_DIRECTION_NONE,
868 - .ipv6ony = {
869 - .checked = false,
870 - .ipv46 = false,
871 - },
872 - .local = {
873 - .family = family,
874 - .protocol = protocol,
875 - },
876 - .remote = {
877 - .family = family,
878 - .protocol = protocol,
879 - },
880 - .uid = UID_UNSET,
881 - };
882 -
958 char *words[32];
959 size_t num_words = quoted_strings_splitter(line, words, 32, is_space);
885 - // char *sl_txt = get_word(words, num_words, 0);
886 - char *local_ip_txt = get_word(words, num_words, 1);
887 - char *local_port_txt = get_word(words, num_words, 2);
888 - char *remote_ip_txt = get_word(words, num_words, 3);
889 - char *remote_port_txt = get_word(words, num_words, 4);
890 - char *state_txt = get_word(words, num_words, 5);
891 - char *tx_queue_txt = get_word(words, num_words, 6);
892 - char *rx_queue_txt = get_word(words, num_words, 7);
893 - char *tr_txt = get_word(words, num_words, 8);
894 - char *tm_when_txt = get_word(words, num_words, 9);
895 - char *retrans_txt = get_word(words, num_words, 10);
896 - char *uid_txt = get_word(words, num_words, 11);
897 - // char *timeout_txt = get_word(words, num_words, 12);
898 - char *inode_txt = get_word(words, num_words, 13);
899 -
900 - if(!local_ip_txt || !local_port_txt || !remote_ip_txt || !remote_port_txt || !state_txt ||
901 - !tx_queue_txt || !rx_queue_txt || !tr_txt || !tm_when_txt || !retrans_txt || !uid_txt || !inode_txt) {
902 - local_sockets_log(ls, "cannot parse ipv4 line No %zu of filename '%s'", counter, filename);
903 - continue;
904 - }
905 -
906 - n.local.port = str2uint32_hex(local_port_txt, NULL);
907 - n.remote.port = str2uint32_hex(remote_port_txt, NULL);
908 - n.state = str2uint32_hex(state_txt, NULL);
909 - n.wqueue = str2uint32_hex(tx_queue_txt, NULL);
910 - n.rqueue = str2uint32_hex(rx_queue_txt, NULL);
911 - n.timer = str2uint32_hex(tr_txt, NULL);
912 - n.expires = str2uint32_hex(tm_when_txt, NULL);
913 - n.retransmits = str2uint32_hex(retrans_txt, NULL);
914 - n.uid = str2uint32_t(uid_txt, NULL);
915 - n.inode = str2uint64_t(inode_txt, NULL);
916 -
917 - if(family == AF_INET) {
918 - n.local.ip.ipv4 = str2uint32_hex(local_ip_txt, NULL);
919 - n.remote.ip.ipv4 = str2uint32_hex(remote_ip_txt, NULL);
920 - }
921 - else if(family == AF_INET6) {
922 - ipv6_to_in6_addr(local_ip_txt, &n.local.ip.ipv6);
923 - ipv6_to_in6_addr(remote_ip_txt, &n.remote.ip.ipv6);
924 - }
925 -
926 - local_sockets_add_socket(ls, &n);
960 + local_sockets_process_proc_line(ls, filename, family, protocol, counter, words, num_words);
961 }
962
963 fclose(fp);
@@ -934,6 +968,59 @@ static inline bool local_sockets_read_proc_net_x(LS_STATE *ls, const char *filen
968 return true;
969 }
970
971 +#define INITIALLY_EXPECTED_PROC_NET_LINES 16384
972 +#define PROC_NET_BYTES_PER_LINE 155 // 105 for IPv4, 155 for IPv6
973 +#define PROC_NET_WORDS_PER_LINE 22
974 +#define INITIALLY_EXPECTED_PROC_NET_WORDS (INITIALLY_EXPECTED_PROC_NET_LINES * PROC_NET_WORDS_PER_LINE)
975 +#define INITIALLY_EXPECTED_PROC_NET_BYTES (INITIALLY_EXPECTED_PROC_NET_LINES * PROC_NET_BYTES_PER_LINE)
976 +
977 +static inline bool local_sockets_read_proc_net_x_procfile(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) {
978 + if(family != AF_INET && family != AF_INET6)
979 + return false;
980 +
981 + procfile_set_adaptive_allocation(true, INITIALLY_EXPECTED_PROC_NET_BYTES, INITIALLY_EXPECTED_PROC_NET_LINES, INITIALLY_EXPECTED_PROC_NET_WORDS);
982 +
983 + bool copy_initial_ff_stats = ls->ff == NULL && ls->stats.ff.memory > 0;
984 + ls->ff = procfile_reopen(ls->ff, filename, ls->ff ? NULL :" :", PROCFILE_FLAG_DEFAULT);
985 +
986 + // we just created ff, copy our old stats to it
987 + if(ls->ff && copy_initial_ff_stats) ls->ff->stats = ls->stats.ff;
988 +
989 + ls->ff = procfile_readall(ls->ff);
990 + if(!ls->ff) return false;
991 +
992 + // get the latest stats from ff;
993 + ls->stats.ff = ls->ff->stats;
994 +
995 + for(size_t l = 1; l < procfile_lines(ls->ff) ;l++) {
996 + size_t w = procfile_linewords(ls->ff, l);
997 + if(!w) continue;
998 + if(w < 14) {
999 + local_sockets_log(ls, "too small line No %zu of filename '%s' (has %zu words)", l, filename, w);
1000 + continue;
1001 + }
1002 +
1003 + char *words[14] = { 0 };
1004 + words[0] = procfile_lineword(ls->ff, l, 0);
1005 + words[1] = procfile_lineword(ls->ff, l, 1);
1006 + words[2] = procfile_lineword(ls->ff, l, 2);
1007 + words[3] = procfile_lineword(ls->ff, l, 3);
1008 + words[4] = procfile_lineword(ls->ff, l, 4);
1009 + words[5] = procfile_lineword(ls->ff, l, 5);
1010 + words[6] = procfile_lineword(ls->ff, l, 6);
1011 + words[7] = procfile_lineword(ls->ff, l, 7);
1012 + words[8] = procfile_lineword(ls->ff, l, 8);
1013 + words[9] = procfile_lineword(ls->ff, l, 9);
1014 + words[10] = procfile_lineword(ls->ff, l, 10);
1015 + words[11] = procfile_lineword(ls->ff, l, 11);
1016 + words[12] = procfile_lineword(ls->ff, l, 12);
1017 + words[13] = procfile_lineword(ls->ff, l, 13);
1018 + local_sockets_process_proc_line(ls, filename, family, protocol, l, words, _countof(words));
1019 + }
1020 +
1021 + return true;
1022 +}
1023 +
1024 // --------------------------------------------------------------------------------------------------------------------
1025
1026 static inline void local_sockets_detect_directions(LS_STATE *ls) {
@@ -1025,31 +1112,33 @@ static inline void local_sockets_init(LS_STATE *ls) {
1112
1113 memset(&ls->stats, 0, sizeof(ls->stats));
1114
1028 -#ifdef HAVE_LIBMNL
1029 - ls->use_mnl = false;
1030 - ls->nl = NULL;
1115 +#if defined(HAVE_LIBMNL)
1116 ls->tmp_protocol = 0;
1032 - local_sockets_libmnl_init(ls);
1117 #endif
1118
1119 +#if defined(LOCAL_SOCKETS_USE_SETNS)
1120 if(ls->config.namespaces && ls->spawn_server == NULL) {
1121 ls->spawn_server = spawn_server_create(SPAWN_SERVER_OPTION_CALLBACK, NULL, local_sockets_spawn_server_callback, 0, NULL);
1122 ls->spawn_server_is_mine = true;
1123 }
1124 else
1125 ls->spawn_server_is_mine = false;
1126 +#endif
1127 }
1128
1129 static inline void local_sockets_cleanup(LS_STATE *ls) {
1130 + if(ls->ff) {
1131 + ls->stats.ff = ls->ff->stats;
1132 + procfile_close(ls->ff);
1133 + ls->ff = NULL;
1134 + }
1135
1136 +#if defined(LOCAL_SOCKETS_USE_SETNS)
1137 if(ls->spawn_server_is_mine) {
1138 spawn_server_destroy(ls->spawn_server);
1139 ls->spawn_server = NULL;
1140 ls->spawn_server_is_mine = false;
1141 }
1050 -
1051 -#ifdef HAVE_LIBMNL
1052 - local_sockets_libmnl_cleanup(ls);
1142 #endif
1143
1144 // free the sockets hashtable data
@@ -1087,19 +1176,6 @@ static inline void local_sockets_cleanup(LS_STATE *ls) {
1176
1177 // --------------------------------------------------------------------------------------------------------------------
1178
1090 -static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) {
1091 -#ifdef HAVE_LIBMNL
1092 - if(!ls->config.no_mnl && ls->nl && ls->use_mnl) {
1093 - ls->use_mnl = local_sockets_libmnl_get_sockets(ls, family, protocol);
1094 -
1095 - if(ls->use_mnl)
1096 - return;
1097 - }
1098 -#endif
1099 -
1100 - local_sockets_read_proc_net_x(ls, filename, family, protocol);
1101 -}
1102 -
1179 static inline void local_sockets_track_time(LS_STATE *ls, const char *name) {
1180 if(!ls->config.report || ls->timings_idx >= _countof(ls->timings))
1181 return;
@@ -1122,6 +1198,60 @@ static inline void local_sockets_track_time(LS_STATE *ls, const char *name) {
1198 }
1199 }
1200
1201 +static void local_sockets_track_time_by_protocol(LS_STATE *ls, bool mnl, uint16_t family, uint16_t protocol) {
1202 + if(mnl) {
1203 + if(family == AF_INET) {
1204 + if(protocol == IPPROTO_TCP)
1205 + local_sockets_track_time(ls, "mnl_read_tcp4");
1206 + else if(protocol == IPPROTO_UDP)
1207 + local_sockets_track_time(ls, "mnl_read_udp4");
1208 + }
1209 + else if(family == AF_INET6) {
1210 + if(protocol == IPPROTO_TCP)
1211 + local_sockets_track_time(ls, "mnl_read_tcp6");
1212 + else if(protocol == IPPROTO_UDP)
1213 + local_sockets_track_time(ls, "mnl_read_udp6");
1214 + }
1215 + else
1216 + local_sockets_track_time(ls, "mnl_read_unknown");
1217 + }
1218 + else {
1219 + if(family == AF_INET) {
1220 + if(protocol == IPPROTO_TCP)
1221 + local_sockets_track_time(ls, "proc_read_tcp4");
1222 + else if(protocol == IPPROTO_UDP)
1223 + local_sockets_track_time(ls, "proc_read_udp4");
1224 + }
1225 + else if(family == AF_INET6) {
1226 + if(protocol == IPPROTO_TCP)
1227 + local_sockets_track_time(ls, "proc_read_tcp6");
1228 + else if(protocol == IPPROTO_UDP)
1229 + local_sockets_track_time(ls, "proc_read_udp6");
1230 + }
1231 + else
1232 + local_sockets_track_time(ls, "proc_read_unknown");
1233 + }
1234 +}
1235 +
1236 +static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) {
1237 +#if defined(HAVE_LIBMNL)
1238 + if(!ls->config.no_mnl) {
1239 + local_sockets_track_time_by_protocol(ls, true, family, protocol);
1240 + if(local_sockets_libmnl_get_sockets(ls, family, protocol))
1241 + return;
1242 +
1243 + // else, do proc
1244 + }
1245 +#endif
1246 +
1247 + local_sockets_track_time_by_protocol(ls, false, family, protocol);
1248 +
1249 + if(ls->config.procfile)
1250 + local_sockets_read_proc_net_x_procfile(ls, filename, family, protocol);
1251 + else
1252 + local_sockets_read_proc_net_x_getline(ls, filename, family, protocol);
1253 +}
1254 +
1255 static inline void local_sockets_read_all_system_sockets(LS_STATE *ls) {
1256 char path[FILENAME_MAX + 1];
1257
@@ -1129,55 +1259,71 @@ static inline void local_sockets_read_all_system_sockets(LS_STATE *ls) {
1259 local_sockets_track_time(ls, "read_namespaces");
1260 snprintfz(path, sizeof(path), "%s/proc/self/ns/net", ls->config.host_prefix);
1261 local_sockets_read_proc_inode_link(ls, path, &ls->proc_self_net_ns_inode, "net");
1132 -
1262 }
1263
1264 if(ls->config.cmdline || ls->config.comm || ls->config.pid || ls->config.namespaces) {
1136 - local_sockets_track_time(ls, "read_proc_pids");
1265 + local_sockets_track_time(ls, "proc_read_pids");
1266 snprintfz(path, sizeof(path), "%s/proc", ls->config.host_prefix);
1267 local_sockets_find_all_sockets_in_proc(ls, path);
1268 }
1269
1270 if(ls->config.tcp4) {
1142 - local_sockets_track_time(ls, "read_tcp4");
1271 snprintfz(path, sizeof(path), "%s/proc/net/tcp", ls->config.host_prefix);
1272 local_sockets_do_family_protocol(ls, path, AF_INET, IPPROTO_TCP);
1273 }
1274
1275 if(ls->config.udp4) {
1148 - local_sockets_track_time(ls, "read_udp4");
1276 snprintfz(path, sizeof(path), "%s/proc/net/udp", ls->config.host_prefix);
1277 local_sockets_do_family_protocol(ls, path, AF_INET, IPPROTO_UDP);
1278 }
1279
1280 if(ls->config.tcp6) {
1154 - local_sockets_track_time(ls, "read_tcp6");
1281 snprintfz(path, sizeof(path), "%s/proc/net/tcp6", ls->config.host_prefix);
1282 local_sockets_do_family_protocol(ls, path, AF_INET6, IPPROTO_TCP);
1283 }
1284
1285 if(ls->config.udp6) {
1160 - local_sockets_track_time(ls, "read_udp6");
1286 snprintfz(path, sizeof(path), "%s/proc/net/udp6", ls->config.host_prefix);
1287 local_sockets_do_family_protocol(ls, path, AF_INET6, IPPROTO_UDP);
1288 }
1289 }
1290
1291 // --------------------------------------------------------------------------------------------------------------------
1292 +// switch namespaces to read namespace sockets
1293 +
1294 +#if defined(LOCAL_SOCKETS_USE_SETNS)
1295
1296 struct local_sockets_child_work {
1297 int fd;
1298 uint64_t net_ns_inode;
1299 };
1300
1173 -static inline void local_sockets_send_to_parent(struct local_socket_state *ls __maybe_unused, struct local_socket *n, void *data) {
1301 +#define LOCAL_SOCKET_TERMINATOR (struct local_socket) { \
1302 + .expires = UINT32_MAX, \
1303 + .timer = UINT8_MAX, \
1304 + .inode = UINT64_MAX, \
1305 + .net_ns_inode = UINT64_MAX, \
1306 +}
1307 +
1308 +static inline bool local_socket_is_terminator(const struct local_socket *n) {
1309 + static const struct local_socket t = LOCAL_SOCKET_TERMINATOR;
1310 + return (n->expires == t.expires &&
1311 + n->timer == t.timer &&
1312 + n->inode == t.inode &&
1313 + n->net_ns_inode == t.net_ns_inode);
1314 +}
1315 +
1316 +static inline void local_sockets_send_to_parent(struct local_socket_state *ls, const struct local_socket *n, void *data) {
1317 struct local_sockets_child_work *cw = data;
1318 int fd = cw->fd;
1319
1177 - if(n->net_ns_inode != cw->net_ns_inode)
1178 - return;
1179 -
1180 - // local_sockets_log(ls, "child is sending inode %"PRIu64" of namespace %"PRIu64, n->inode, n->net_ns_inode);
1320 + if(!local_socket_is_terminator(n)) {
1321 + ls->stats.errors_encountered = 0;
1322 +// local_sockets_log(
1323 +// ls,
1324 +// "child is sending inode %"PRIu64" of namespace %"PRIu64", from namespace %"PRIu64" for pid %d",
1325 +// n->inode, n->net_ns_inode, ls->proc_self_net_ns_inode, ls->ns_state.net_ns_pid);
1326 + }
1327
1328 if(write(fd, n, sizeof(*n)) != sizeof(*n))
1329 local_sockets_log(ls, "failed to write local socket to pipe");
@@ -1192,8 +1338,14 @@ static inline void local_sockets_send_to_parent(struct local_socket_state *ls __
1338 }
1339
1340 static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request) {
1341 + static const struct local_socket terminator = LOCAL_SOCKET_TERMINATOR;
1342 +
1343 + struct local_sockets_ns_req *req = (struct local_sockets_ns_req *)request->data;
1344 +
1345 LS_STATE ls = { 0 };
1196 - ls.config = *((struct local_sockets_config *)request->data);
1346 + ls.config = req->config;
1347 + ls.ns_state = req->ns_state;
1348 + ls.ns_state.nl_seq += gettid_uncached() * 10;
1349
1350 // we don't need these inside namespaces
1351 ls.config.cmdline = false;
@@ -1201,9 +1353,13 @@ static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request) {
1353 ls.config.pid = false;
1354 ls.config.namespaces = false;
1355
1356 +#if !defined(USE_LIBMNL_AFTER_SETNS)
1357 + ls.config.no_mnl = true; // disable mnl since this collects all sockets from the entire system
1358 +#endif
1359 +
1360 // initialize local sockets
1361 local_sockets_init(&ls);
1206 -
1362 + ls.proc_self_net_ns_inode = ls.ns_state.net_ns_inode;
1363 ls.config.host_prefix = ""; // we need the /proc of the container
1364
1365 struct local_sockets_child_work cw = {
@@ -1213,7 +1369,6 @@ static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request) {
1369
1370 ls.config.cb = local_sockets_send_to_parent;
1371 ls.config.data = &cw;
1216 - ls.proc_self_net_ns_inode = ls.config.net_ns_inode;
1372
1373 // switch namespace using the custom fd passed via the spawn server
1374 if (setns(request->fds[3], CLONE_NEWNET) == -1) {
@@ -1221,6 +1376,9 @@ static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request) {
1376 return EXIT_FAILURE;
1377 }
1378
1379 + // close the custom fd
1380 + close(request->fds[3]); request->fds[3] = -1;
1381 +
1382 // read all sockets from /proc
1383 local_sockets_read_all_system_sockets(&ls);
1384
@@ -1228,10 +1386,7 @@ static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request) {
1386 local_sockets_foreach_local_socket_call_cb(&ls);
1387
1388 // send the terminating socket
1231 - struct local_socket zero = {
1232 - .net_ns_inode = ls.config.net_ns_inode,
1233 - };
1234 - local_sockets_send_to_parent(&ls, &zero, &cw);
1389 + local_sockets_send_to_parent(&ls, &terminator, &cw);
1390
1391 local_sockets_cleanup(&ls);
1392
@@ -1246,6 +1401,8 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st
1401 int fd = open(filename, O_RDONLY | O_CLOEXEC);
1402 if (fd == -1) {
1403 local_sockets_log(ls, "cannot open file '%s'", filename);
1404 + if(ls->config.report)
1405 + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED);
1406 return false;
1407 }
1408
@@ -1253,28 +1410,46 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st
1410 if (fstat(fd, &statbuf) == -1) {
1411 close(fd);
1412 local_sockets_log(ls, "failed to get file statistics for '%s'", filename);
1413 + if(ls->config.report)
1414 + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED);
1415 return false;
1416 }
1417
1418 if (statbuf.st_ino != ps->net_ns_inode) {
1419 close(fd);
1420 local_sockets_log(ls, "pid %d is not in the wanted network namespace", ps->pid);
1421 + if(ls->config.report)
1422 + __atomic_add_fetch(&ls->stats.namespaces_invalid, 1, __ATOMIC_RELAXED);
1423 return false;
1424 }
1425
1426 if(ls->spawn_server == NULL) {
1427 close(fd);
1428 local_sockets_log(ls, "spawn server is not available");
1429 + if(ls->config.report)
1430 + __atomic_add_fetch(&ls->stats.namespaces_forks_failed, 1, __ATOMIC_RELAXED);
1431 return false;
1432 }
1433
1271 - struct local_sockets_config config = ls->config;
1272 - config.net_ns_inode = ps->net_ns_inode;
1273 - SPAWN_INSTANCE *si = spawn_server_exec(ls->spawn_server, STDERR_FILENO, fd, NULL, &config, sizeof(config), SPAWN_INSTANCE_TYPE_CALLBACK);
1434 + struct local_sockets_ns_req req = {
1435 + .config = ls->config,
1436 + .ns_state = ls->ns_state,
1437 + };
1438 + req.ns_state.net_ns_pid = ps->pid;
1439 + req.ns_state.net_ns_inode = ps->net_ns_inode;
1440 +
1441 + SPAWN_INSTANCE *si = spawn_server_exec(ls->spawn_server, STDERR_FILENO, fd, NULL, &req, sizeof(req), SPAWN_INSTANCE_TYPE_CALLBACK);
1442 close(fd); fd = -1;
1443
1444 + if(ls->config.report)
1445 + __atomic_add_fetch(&ls->stats.namespaces_forks_attempted, 1, __ATOMIC_RELAXED);
1446 +
1447 if(si == NULL) {
1448 local_sockets_log(ls, "cannot create spawn instance");
1449 +
1450 + if(ls->config.report)
1451 + __atomic_add_fetch(&ls->stats.namespaces_forks_failed, 1, __ATOMIC_RELAXED);
1452 +
1453 return false;
1454 }
1455
@@ -1299,13 +1474,12 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st
1474
1475 received++;
1476
1302 - struct local_socket zero = {
1303 - .net_ns_inode = ps->net_ns_inode,
1304 - };
1305 - if(memcmp(&buf, &zero, sizeof(buf)) == 0) {
1306 - // the terminator
1477 + if(local_socket_is_terminator(&buf))
1478 + // the child finished
1479 break;
1308 - }
1480 +
1481 + // overwrite the net_ns_inode we receive
1482 + buf.net_ns_inode = ps->net_ns_inode;
1483
1484 spinlock_lock(&ls->spinlock);
1485
@@ -1313,9 +1487,13 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st
1487 LOCAL_SOCKET *n = SIMPLE_HASHTABLE_SLOT_DATA(sl);
1488 if(n) {
1489 string_freez(buf.cmdline);
1490 +
1491 // local_sockets_log(ls,
1492 // "ns inode %" PRIu64" (comm: '%s', pid: %u, ns: %"PRIu64") already exists in hashtable (comm: '%s', pid: %u, ns: %"PRIu64") - ignoring duplicate",
1493 // buf.inode, buf.comm, buf.pid, buf.net_ns_inode, n->comm, n->pid, n->net_ns_inode);
1494 +
1495 + if(ls->config.report)
1496 + __atomic_add_fetch(&ls->stats.namespaces_sockets_existing, 1, __ATOMIC_RELAXED);
1497 }
1498 else {
1499 n = aral_mallocz(ls->local_socket_aral);
@@ -1323,12 +1501,19 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st
1501 simple_hashtable_set_slot_LOCAL_SOCKET(&ls->sockets_hashtable, sl, n->inode, n);
1502
1503 local_sockets_index_listening_port(ls, n);
1504 +
1505 + if(ls->config.report)
1506 + __atomic_add_fetch(&ls->stats.namespaces_sockets_new, 1, __ATOMIC_RELAXED);
1507 }
1508
1509 spinlock_unlock(&ls->spinlock);
1510 }
1511
1512 spawn_server_exec_kill(ls->spawn_server, si);
1513 +
1514 + if(ls->config.report && received == 0)
1515 + __atomic_add_fetch(&ls->stats.namespaces_forks_unresponsive, 1, __ATOMIC_RELAXED);
1516 +
1517 return received > 0;
1518 }
1519
@@ -1385,6 +1570,7 @@ static inline void local_sockets_namespaces(LS_STATE *ls) {
1570 const uint64_t inode = (uint64_t)SIMPLE_HASHTABLE_SLOT_DATA(sl);
1571
1572 if(inode == ls->proc_self_net_ns_inode)
1573 + // skip our own namespace, we already have them
1574 continue;
1575
1576 spinlock_unlock(&ls->spinlock);
@@ -1421,6 +1607,100 @@ static inline void local_sockets_namespaces(LS_STATE *ls) {
1607 }
1608 }
1609
1610 +#endif // LOCAL_SOCKETS_USE_SETNS
1611 +
1612 +// --------------------------------------------------------------------------------------------------------------------
1613 +// read namespace sockets from the host's /proc
1614 +
1615 +#if !defined(LOCAL_SOCKETS_USE_SETNS)
1616 +
1617 +static inline bool local_sockets_namespaces_from_proc_with_pid(LS_STATE *ls, struct pid_socket *ps) {
1618 + char filename[1024];
1619 + snprintfz(filename, sizeof(filename), "%s/proc/%d/ns/net", ls->config.host_prefix, ps->pid);
1620 +
1621 + // verify the pid is in the target namespace
1622 + int fd = open(filename, O_RDONLY | O_CLOEXEC);
1623 + if (fd == -1) {
1624 + local_sockets_log(ls, "cannot open file '%s'", filename);
1625 + if(ls->config.report)
1626 + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED);
1627 + return false;
1628 + }
1629 +
1630 + struct stat statbuf;
1631 + if (fstat(fd, &statbuf) == -1) {
1632 + close(fd);
1633 + local_sockets_log(ls, "failed to get file statistics for '%s'", filename);
1634 + if(ls->config.report)
1635 + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED);
1636 + return false;
1637 + }
1638 +
1639 + if (statbuf.st_ino != ps->net_ns_inode) {
1640 + close(fd);
1641 + local_sockets_log(ls, "pid %d is not in the wanted network namespace", ps->pid);
1642 + if(ls->config.report)
1643 + __atomic_add_fetch(&ls->stats.namespaces_invalid, 1, __ATOMIC_RELAXED);
1644 + return false;
1645 + }
1646 +
1647 + char path[FILENAME_MAX + 1];
1648 +
1649 + if(ls->config.tcp4) {
1650 + snprintfz(path, sizeof(path), "%s/proc/%d/net/tcp", ls->config.host_prefix, ps->pid);
1651 + if(!local_sockets_read_proc_net_x(ls, path, AF_INET, IPPROTO_TCP))
1652 + return false;
1653 + }
1654 +
1655 + if(ls->config.udp4) {
1656 + snprintfz(path, sizeof(path), "%s/proc/%d/net/udp", ls->config.host_prefix, ps->pid);
1657 + if(!local_sockets_read_proc_net_x(ls, path, AF_INET, IPPROTO_UDP))
1658 + return false;
1659 + }
1660 +
1661 + if(ls->config.tcp6) {
1662 + snprintfz(path, sizeof(path), "%s/proc/%d/net/tcp6", ls->config.host_prefix, ps->pid);
1663 + if(!local_sockets_read_proc_net_x(ls, path, AF_INET6, IPPROTO_TCP))
1664 + return false;
1665 + }
1666 +
1667 + if(ls->config.udp6) {
1668 + snprintfz(path, sizeof(path), "%s/proc/%d/net/udp6", ls->config.host_prefix, ps->pid);
1669 + if(!local_sockets_read_proc_net_x(ls, path, AF_INET6, IPPROTO_UDP))
1670 + return false;
1671 + }
1672 +
1673 + return true;
1674 +}
1675 +
1676 +static inline void local_sockets_namespaces_from_proc(LS_STATE *ls) {
1677 + for(SIMPLE_HASHTABLE_SLOT_NET_NS *sl = simple_hashtable_first_read_only_NET_NS(&ls->ns_hashtable);
1678 + sl;
1679 + sl = simple_hashtable_next_read_only_NET_NS(&ls->ns_hashtable, sl)) {
1680 + const uint64_t inode = (uint64_t)SIMPLE_HASHTABLE_SLOT_DATA(sl);
1681 +
1682 + if (inode == ls->proc_self_net_ns_inode)
1683 + // skip our own namespace, we already have them
1684 + continue;
1685 +
1686 + ls->stats.namespaces_found++;
1687 +
1688 + for(SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl_pid = simple_hashtable_first_read_only_PID_SOCKET(&ls->pid_sockets_hashtable) ;
1689 + sl_pid ;
1690 + sl_pid = simple_hashtable_next_read_only_PID_SOCKET(&ls->pid_sockets_hashtable, sl_pid)) {
1691 + struct pid_socket *ps = SIMPLE_HASHTABLE_SLOT_DATA(sl_pid);
1692 + if(!ps || ps->net_ns_inode != inode) continue;
1693 +
1694 + // now we have a pid that has the same namespace inode
1695 +
1696 + if(local_sockets_namespaces_from_proc_with_pid(ls, ps))
1697 + break;
1698 + }
1699 + }
1700 +}
1701 +
1702 +#endif
1703 +
1704 // --------------------------------------------------------------------------------------------------------------------
1705
1706 static inline void local_sockets_process(LS_STATE *ls) {
@@ -1438,7 +1718,11 @@ static inline void local_sockets_process(LS_STATE *ls) {
1718 // check all socket namespaces
1719 if(ls->config.namespaces) {
1720 local_sockets_track_time(ls, "switch_namespaces");
1721 +#if defined(LOCAL_SOCKETS_USE_SETNS)
1722 local_sockets_namespaces(ls);
1723 +#else
1724 + local_sockets_namespaces_from_proc(ls);
1725 +#endif
1726 }
1727
1728 // detect the directions of the sockets
@@ -1456,7 +1740,7 @@ static inline void local_sockets_process(LS_STATE *ls) {
1740 local_sockets_cleanup(ls);
1741 }
1742
1459 -static inline void ipv6_address_to_txt(struct in6_addr *in6_addr, char *dst) {
1743 +static inline void ipv6_address_to_txt(const struct in6_addr *in6_addr, char *dst) {
1744 struct sockaddr_in6 sa = { 0 };
1745
1746 sa.sin6_family = AF_INET6;
src/libnetdata/procfile/procfile.c
+51 -10
@@ -10,14 +10,23 @@
10
11 int procfile_open_flags = O_RDONLY | O_CLOEXEC;
12
13 -int procfile_adaptive_initial_allocation = 0;
14 -
13 // if adaptive allocation is set, these store the
14 // max values we have seen so far
17 -size_t procfile_max_lines = PFLINES_INCREASE_STEP;
18 -size_t procfile_max_words = PFWORDS_INCREASE_STEP;
19 -size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER;
20 -
15 +static bool procfile_adaptive_initial_allocation = false;
16 +static size_t procfile_max_lines = PFLINES_INCREASE_STEP;
17 +static size_t procfile_max_words = PFWORDS_INCREASE_STEP;
18 +static size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER;
19 +
20 +void procfile_set_adaptive_allocation(bool enable, size_t bytes, size_t lines, size_t words) {
21 + procfile_adaptive_initial_allocation = enable;
22 +
23 + if(bytes > procfile_max_allocation)
24 + procfile_max_allocation = bytes;
25 + if(lines > procfile_max_lines)
26 + procfile_max_lines = lines;
27 + if(words > procfile_max_words)
28 + procfile_max_words = words;
29 +}
30
31 // ----------------------------------------------------------------------------
32
@@ -59,6 +68,8 @@ static inline void procfile_words_add(procfile *ff, char *str) {
68
69 ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + wanted) * sizeof(char *));
70 fw->size += wanted;
71 + ff->stats.memory += wanted * sizeof(char *);
72 + ff->stats.resizes++;
73 }
74
75 fw->words[fw->len++] = str;
@@ -92,7 +103,7 @@ static inline void procfile_words_free(pfwords *fw) {
103 // An array of lines
104
105 NEVERNULL
95 -static inline size_t *procfile_lines_add(procfile *ff) {
106 +static inline uint32_t *procfile_lines_add(procfile *ff) {
107 // netdata_log_debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word);
108
109 pflines *fl = ff->lines;
@@ -104,6 +115,8 @@ static inline size_t *procfile_lines_add(procfile *ff) {
115
116 ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + wanted) * sizeof(ffline));
117 fl->size += wanted;
118 + ff->stats.memory += wanted * sizeof(ffline);
119 + ff->stats.resizes++;
120 }
121
122 ffline *ffl = &fl->lines[fl->len++];
@@ -168,7 +181,7 @@ static void procfile_parser(procfile *ff) {
181 char quote = 0; // the quote character - only when in quoted string
182 size_t opened = 0; // counts the number of open parenthesis
183
171 - size_t *line_words = procfile_lines_add(ff);
184 + uint32_t *line_words = procfile_lines_add(ff);
185
186 while(s < e) {
187 PF_CHAR_TYPE ct = separators[(unsigned char)(*s)];
@@ -279,6 +292,8 @@ static void procfile_parser(procfile *ff) {
292 }
293
294 procfile *procfile_readall(procfile *ff) {
295 + if(!ff) return NULL;
296 +
297 // netdata_log_debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename);
298
299 ff->len = 0; // zero the used size
@@ -295,9 +310,12 @@ procfile *procfile_readall(procfile *ff) {
310 netdata_log_debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted);
311 ff = reallocz(ff, sizeof(procfile) + ff->size + wanted);
312 ff->size += wanted;
313 + ff->stats.memory += wanted;
314 + ff->stats.resizes++;
315 }
316
300 - netdata_log_debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s));
317 + // netdata_log_info("Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s));
318 + ff->stats.reads++;
319 r = read(ff->fd, &ff->data[s], ff->size - s);
320 if(unlikely(r == -1)) {
321 if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) collector_error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd);
@@ -307,6 +325,9 @@ procfile *procfile_readall(procfile *ff) {
325 return NULL;
326 }
327
328 + if((ssize_t)ff->stats.max_read_size < r)
329 + ff->stats.max_read_size = r;
330 +
331 ff->len += r;
332 }
333
@@ -329,6 +350,17 @@ procfile *procfile_readall(procfile *ff) {
350 if(unlikely(ff->words->len > procfile_max_words)) procfile_max_words = ff->words->len;
351 }
352
353 + if(ff->stats.max_source_bytes < ff->len)
354 + ff->stats.max_source_bytes = ff->len;
355 +
356 + if(ff->stats.max_lines < ff->lines->len)
357 + ff->stats.max_lines = ff->lines->len;
358 +
359 + if(ff->stats.max_words < ff->words->len)
360 + ff->stats.max_words = ff->words->len;
361 +
362 + ff->stats.total_read_bytes += ff->len;
363 +
364 // netdata_log_debug(D_PROCFILE, "File '%s' updated.", ff->filename);
365 return ff;
366 }
@@ -433,10 +465,18 @@ procfile *procfile_open(const char *filename, const char *separators, uint32_t f
465 ff->size = size;
466 ff->len = 0;
467 ff->flags = flags;
468 + ff->stats.opens = 1;
469 + ff->stats.reads = ff->stats.resizes = 0;
470 + ff->stats.max_lines = ff->stats.max_words = ff->stats.max_source_bytes = 0;
471 + ff->stats.total_read_bytes = ff->stats.max_read_size = 0;
472
473 ff->lines = procfile_lines_create();
474 ff->words = procfile_words_create();
475
476 + ff->stats.memory = sizeof(procfile) + size +
477 + (sizeof(pflines) + ff->lines->size * sizeof(ffline)) +
478 + (sizeof(pfwords) + ff->words->size * sizeof(char *));
479 +
480 procfile_set_separators(ff, separators);
481
482 netdata_log_debug(D_PROCFILE, "File '%s' opened.", filename);
@@ -456,6 +496,7 @@ procfile *procfile_reopen(procfile *ff, const char *filename, const char *separa
496 procfile_close(ff);
497 return NULL;
498 }
499 + ff->stats.opens++;
500
501 // netdata_log_info("PROCFILE: opened '%s' on fd %d", filename, ff->fd);
502
@@ -483,7 +524,7 @@ void procfile_print(procfile *ff) {
524 for(l = 0; likely(l < lines) ;l++) {
525 size_t words = procfile_linewords(ff, l);
526
486 - netdata_log_debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words);
527 + netdata_log_debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, (size_t)ff->lines->lines[l].first, (size_t)ff->lines->lines[l].words);
528
529 size_t w;
530 for(w = 0; likely(w < words) ;w++) {
src/libnetdata/procfile/procfile.h
+21 -6
@@ -19,9 +19,8 @@ typedef struct {
19 // An array of lines
20
21 typedef struct {
22 - size_t words; // how many words this line has
23 - size_t first; // the id of the first word of this line
24 - // in the words array
22 + uint32_t words; // how many words this line has
23 + uint32_t first; // the id of the first word of this line in the words array
24 } ffline;
25
26 typedef struct {
@@ -35,7 +34,7 @@ typedef struct {
34 // The procfile
35
36 #define PROCFILE_FLAG_DEFAULT 0x00000000 // To store inside `collector.log`
38 -#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001 // Do not store nothing
37 +#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001 // Do not log anything
38 #define PROCFILE_FLAG_ERROR_ON_ERROR_LOG 0x00000002 // Store inside `error.log`
39
40 typedef enum __attribute__ ((__packed__)) procfile_separator {
@@ -47,7 +46,22 @@ typedef enum __attribute__ ((__packed__)) procfile_separator {
46 PF_CHAR_IS_CLOSE
47 } PF_CHAR_TYPE;
48
49 +struct procfile_stats {
50 + size_t opens;
51 + size_t reads;
52 + size_t resizes;
53 + size_t memory;
54 + size_t total_read_bytes;
55 + size_t max_source_bytes;
56 + size_t max_lines;
57 + size_t max_words;
58 + size_t max_read_size;
59 +};
60 +
61 +
62 typedef struct procfile {
63 + // this structure is malloc'd (you need to initialize it at procfile_open()
64 +
65 char *filename; // not populated until procfile_filename() is called
66 uint32_t flags;
67 int fd; // the file descriptor
@@ -56,6 +70,7 @@ typedef struct procfile {
70 pflines *lines;
71 pfwords *words;
72 PF_CHAR_TYPE separators[256];
73 + struct procfile_stats stats;
74 char data[]; // allocated buffer to keep file contents
75 } procfile;
76
@@ -85,8 +100,8 @@ char *procfile_filename(procfile *ff);
100 // set to the O_XXXX flags, to have procfile_open and procfile_reopen use them when opening proc files
101 extern int procfile_open_flags;
102
88 -// set this to 1, to have procfile adapt its initial buffer allocation to the max allocation used so far
89 -extern int procfile_adaptive_initial_allocation;
103 +// call this with true and the expected initial sizes to allow procfile learn the sizes needed
104 +void procfile_set_adaptive_allocation(bool enable, size_t bytes, size_t lines, size_t words);
105
106 // return the number of lines present
107 #define procfile_lines(ff) ((ff)->lines->len)
src/libnetdata/spawn_server/spawn_server_nofork.c
+58 -68
@@ -48,60 +48,6 @@ static int connect_to_spawn_server(const char *path, bool log) {
48 return sock;
49 }
50
51 -// --------------------------------------------------------------------------------------------------------------------
52 -// the child created by the spawn server
53 -
54 -static void spawn_server_run_child(SPAWN_SERVER *server, SPAWN_REQUEST *rq) {
55 - // close the server sockets;
56 - close(server->sock); server->sock = -1;
57 - if(server->pipe[0] != -1) { close(server->pipe[0]); server->pipe[0] = -1; }
58 - if(server->pipe[1] != -1) { close(server->pipe[1]); server->pipe[1] = -1; }
59 -
60 - // close all open file descriptors of the parent, but keep ours
61 - os_close_all_non_std_open_fds_except(rq->fds, 4, 0);
62 - nd_log_reopen_log_files_for_spawn_server();
63 -
64 - // set the process name
65 - os_setproctitle("spawn-child", server->argc, server->argv);
66 -
67 - // get the fds from the request
68 - int stdin_fd = rq->fds[0];
69 - int stdout_fd = rq->fds[1];
70 - int stderr_fd = rq->fds[2];
71 - int custom_fd = rq->fds[3]; (void)custom_fd;
72 -
73 - // change stdio fds to the ones in the request
74 - if (dup2(stdin_fd, STDIN_FILENO) == -1) {
75 - nd_log(NDLS_COLLECTORS, NDLP_ERR,
76 - "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s",
77 - stdin_fd, rq->request_id, rq->cmdline);
78 - exit(EXIT_FAILURE);
79 - }
80 - if (dup2(stdout_fd, STDOUT_FILENO) == -1) {
81 - nd_log(NDLS_COLLECTORS, NDLP_ERR,
82 - "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s",
83 - stdout_fd, rq->request_id, rq->cmdline);
84 - exit(EXIT_FAILURE);
85 - }
86 - if (dup2(stderr_fd, STDERR_FILENO) == -1) {
87 - nd_log(NDLS_COLLECTORS, NDLP_ERR,
88 - "SPAWN SERVER: cannot dup2(%d) stderr of request No %zu: %s",
89 - stderr_fd, rq->request_id, rq->cmdline);
90 - exit(EXIT_FAILURE);
91 - }
92 -
93 - // close the excess fds
94 - close(stdin_fd); stdin_fd = rq->fds[0] = STDIN_FILENO;
95 - close(stdout_fd); stdout_fd = rq->fds[1] = STDOUT_FILENO;
96 - close(stderr_fd); stderr_fd = rq->fds[2] = STDERR_FILENO;
97 -
98 - // overwrite the process environment
99 - environ = (char **)rq->envp;
100 -
101 - // run the callback and return its code
102 - exit(server->cb(rq));
103 -}
104 -
51 // --------------------------------------------------------------------------------------------------------------------
52 // Encoding and decoding of spawn server request argv type of data
53
@@ -364,8 +310,54 @@ static bool spawn_server_run_callback(SPAWN_SERVER *server __maybe_unused, SPAWN
310 else if (pid == 0) {
311 // the child
312
367 - spawn_server_run_child(server, rq);
368 - exit(63);
313 + // close the server sockets;
314 + close(server->sock); server->sock = -1;
315 + if(server->pipe[0] != -1) { close(server->pipe[0]); server->pipe[0] = -1; }
316 + if(server->pipe[1] != -1) { close(server->pipe[1]); server->pipe[1] = -1; }
317 +
318 + // set the process name
319 + os_setproctitle("spawn-callback", server->argc, server->argv);
320 +
321 + // close all open file descriptors of the parent, but keep ours
322 + os_close_all_non_std_open_fds_except(rq->fds, 4, 0);
323 + nd_log_reopen_log_files_for_spawn_server("spawn-callback");
324 +
325 + // get the fds from the request
326 + int stdin_fd = rq->fds[0];
327 + int stdout_fd = rq->fds[1];
328 + int stderr_fd = rq->fds[2];
329 + int custom_fd = rq->fds[3]; (void)custom_fd;
330 +
331 + // change stdio fds to the ones in the request
332 + if (dup2(stdin_fd, STDIN_FILENO) == -1) {
333 + nd_log(NDLS_COLLECTORS, NDLP_ERR,
334 + "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s",
335 + stdin_fd, rq->request_id, rq->cmdline);
336 + exit(EXIT_FAILURE);
337 + }
338 + if (dup2(stdout_fd, STDOUT_FILENO) == -1) {
339 + nd_log(NDLS_COLLECTORS, NDLP_ERR,
340 + "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s",
341 + stdout_fd, rq->request_id, rq->cmdline);
342 + exit(EXIT_FAILURE);
343 + }
344 + if (dup2(stderr_fd, STDERR_FILENO) == -1) {
345 + nd_log(NDLS_COLLECTORS, NDLP_ERR,
346 + "SPAWN SERVER: cannot dup2(%d) stderr of request No %zu: %s",
347 + stderr_fd, rq->request_id, rq->cmdline);
348 + exit(EXIT_FAILURE);
349 + }
350 +
351 + // close the excess fds
352 + close(stdin_fd); stdin_fd = rq->fds[0] = STDIN_FILENO;
353 + close(stdout_fd); stdout_fd = rq->fds[1] = STDOUT_FILENO;
354 + close(stderr_fd); stderr_fd = rq->fds[2] = STDERR_FILENO;
355 +
356 + // overwrite the process environment
357 + environ = (char **)rq->envp;
358 +
359 + // run the callback and return its code
360 + exit(server->cb(rq));
361 }
362
363 // the parent
@@ -834,7 +826,7 @@ static void posix_unmask_sigchld_on_thread(void) {
826 "SPAWN SERVER: cannot unmask SIGCHLD");
827 }
828
837 -static void spawn_server_event_loop(SPAWN_SERVER *server) {
829 +static int spawn_server_event_loop(SPAWN_SERVER *server) {
830 int pipe_fd = server->pipe[1];
831 close(server->pipe[0]); server->pipe[0] = -1;
832
@@ -847,13 +839,13 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) {
839 sa.sa_flags = SA_RESTART | SA_NOCLDSTOP;
840 if (sigaction(SIGCHLD, &sa, NULL) == -1) {
841 nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: sigaction() failed for SIGCHLD");
850 - exit(1);
842 + return 1;
843 }
844
845 sa.sa_handler = spawn_server_sigterm_handler;
846 if (sigaction(SIGTERM, &sa, NULL) == -1) {
847 nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: sigaction() failed for SIGTERM");
856 - exit(1);
848 + return 1;
849 }
850
851 struct status_report sr = {
@@ -864,7 +856,7 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) {
856 };
857 if (write(pipe_fd, &sr, sizeof(sr)) != sizeof(sr)) {
858 nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: failed to write initial status report.");
867 - exit(1);
859 + return 1;
860 }
861
862 struct pollfd fds[2];
@@ -927,7 +919,7 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) {
919 // nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: all %zu children finished", killed);
920 }
921
930 - exit(1);
922 + return 0;
923 }
924
925 // --------------------------------------------------------------------------------------------------------------------
@@ -1073,16 +1065,14 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name
1065 if (pid == 0) {
1066 // the child - the spawn server
1067
1076 - {
1077 - char buf[15];
1078 - snprintfz(buf, sizeof(buf), "spawn-%s", server->name);
1079 - os_setproctitle(buf, server->argc, server->argv);
1080 - }
1068 + char buf[16];
1069 + snprintfz(buf, sizeof(buf), "spawn-%s", server->name);
1070 + os_setproctitle(buf, server->argc, server->argv);
1071
1072 replace_stdio_with_dev_null();
1073 os_close_all_non_std_open_fds_except((int[]){ server->sock, server->pipe[1] }, 2, 0);
1084 - nd_log_reopen_log_files_for_spawn_server();
1085 - spawn_server_event_loop(server);
1074 + nd_log_reopen_log_files_for_spawn_server(buf);
1075 + exit(spawn_server_event_loop(server));
1076 }
1077 else if (pid > 0) {
1078 // the parent