@cryptotaxi247 / netdata-1 / commits / 6c3e9b1f2

local-listeners without libmnl (#18759)

allow disabling libmnl; collect timing statistics and print report

Costa Tsaousis committed Oct 11, 2024 at 20:55 UTC 6c3e9b1f29c411054624bedeb38a71fc58225943

5 files changed +118 -16

src/collectors/utils/local_listeners.c

+35

@@ -106,6 +106,8 @@ int main(int argc, char **argv) {

106 .comm = false,

107 .namespaces = true,

108 .tcp_info = false,

109 + .no_mnl = false,

110 + .report = false,

111

112 .max_errors = 10,

113 .max_concurrent_namespaces = 10,

@@ -158,6 +160,9 @@ int main(int argc, char **argv) {

160 " Option 'debug' enables all sources and all directions and provides\n"

161 " a full dump of current sockets.\n"

162 "\n"

163 + " Option 'report' reports timings per step while collecting and processing\n"

164 + " system information.\n"

165 + "\n"

166 " DIRECTION DETECTION\n"

167 " The program detects the direction of the sockets using these rules:\n"

168 "\n"

@@ -285,6 +290,14 @@ int main(int argc, char **argv) {

290 ls.config.namespaces = positive;

291 // fprintf(stderr, "%s namespaces\n", positive ? "enabling" : "disabling");

292 }

293 + else if (strcmp("mnl", s) == 0) {

294 + ls.config.no_mnl = !positive;

295 + // fprintf(stderr, "%s mnl\n", positive ? "enabling" : "disabling");

296 + }

297 + else if (strcmp("report", s) == 0) {

298 + ls.config.report = positive;

299 + // fprintf(stderr, "%s report\n", positive ? "enabling" : "disabling");

300 + }

301 else {

302 fprintf(stderr, "Unknown parameter %s\n", s);

303 exit(1);

@@ -312,5 +325,27 @@ int main(int argc, char **argv) {

325 fprintf(stderr, "CPU Usage %llu user, %llu system, %llu total, %zu namespaces, %zu nl requests (without namespaces)\n", user, system, total, ls.stats.namespaces_found, ls.stats.mnl_sends);

326 }

327

328 + if(ls.config.report) {

329 + fprintf(stderr, "\nTIMINGS REPORT:\n");

330 + char buf[100];

331 + usec_t total_ut = 0;

332 + for(size_t i = 0; i < _countof(ls.timings) ;i++) {

333 + if (!ls.timings[i].end_ut) continue;

334 + usec_t dt_ut = ls.timings[i].end_ut - ls.timings[i].start_ut;

335 + total_ut += dt_ut;

336 + }

337 +

338 + for(size_t i = 0; i < _countof(ls.timings) ;i++) {

339 + if(!ls.timings[i].end_ut) continue;

340 + usec_t dt_ut = ls.timings[i].end_ut - ls.timings[i].start_ut;

341 + double percent = (100.0 * (double)dt_ut) / (double)total_ut;

342 + duration_snprintf(buf, sizeof(buf), (int64_t)dt_ut, "us", true);

343 + fprintf(stderr, "%20s: %6.2f%% %s\n", ls.timings[i].name, percent, buf);

344 + }

345 +

346 + duration_snprintf(buf, sizeof(buf), (int64_t)total_ut, "us", true);

347 + fprintf(stderr, "%20s: %6.2f%% %s\n", "TOTAL", 100.0, buf);

348 + }

349 +

350 return 0;

351 }

src/libnetdata/maps/local-sockets.h

+73 -14

@@ -5,6 +5,10 @@

6 #include "libnetdata/libnetdata.h"

8 +#ifndef _countof

9 +#define _countof(x) (sizeof(x) / sizeof(*(x)))

10 +#endif

11 +

12 #ifdef HAVE_LIBMNL

13 #include <linux/rtnetlink.h>

14 #include <linux/inet_diag.h>

@@ -80,6 +84,8 @@ struct local_sockets_config {

84 bool uid;

85 bool namespaces;

86 bool tcp_info;

87 + bool no_mnl;

88 + bool report;

90 size_t max_errors;

91 size_t max_concurrent_namespaces;

@@ -93,6 +99,12 @@ struct local_sockets_config {

99 uint64_t net_ns_inode;

100 };

101

102 +struct timing_work {

103 + usec_t start_ut;

104 + usec_t end_ut;

105 + const char *name;

106 +};

107 +

108 typedef struct local_socket_state {

109 struct local_sockets_config config;

110

@@ -107,11 +119,14 @@ typedef struct local_socket_state {

119 size_t errors_encountered;

120 } stats;

121

122 + size_t timings_idx;

123 + struct timing_work timings[20];

124 +

125 bool spawn_server_is_mine;

126 SPAWN_SERVER *spawn_server;

127

128 #ifdef HAVE_LIBMNL

114 - bool use_nl;

129 + bool use_mnl;

130 struct mnl_socket *nl;

131 uint16_t tmp_protocol;

132 #endif

@@ -671,26 +686,28 @@ static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) {

686 #ifdef HAVE_LIBMNL

687

688 static inline void local_sockets_libmnl_init(LS_STATE *ls) {

689 + if(ls->config.no_mnl) return;

690 +

691 ls->nl = mnl_socket_open(NETLINK_INET_DIAG);

692 if (ls->nl == NULL) {

693 local_sockets_log(ls, "cannot open libmnl netlink socket");

677 - ls->use_nl = false;

694 + ls->use_mnl = false;

695 }

696 else if (mnl_socket_bind(ls->nl, 0, MNL_SOCKET_AUTOPID) < 0) {

697 local_sockets_log(ls, "cannot bind libmnl netlink socket");

698 mnl_socket_close(ls->nl);

699 ls->nl = NULL;

683 - ls->use_nl = false;

700 + ls->use_mnl = false;

701 }

702 else

686 - ls->use_nl = true;

703 + ls->use_mnl = true;

704 }

705

706 static inline void local_sockets_libmnl_cleanup(LS_STATE *ls) {

707 if(ls->nl) {

708 mnl_socket_close(ls->nl);

709 ls->nl = NULL;

693 - ls->use_nl = false;

710 + ls->use_mnl = false;

711 }

712 }

713

@@ -1009,7 +1026,7 @@ static inline void local_sockets_init(LS_STATE *ls) {

1026 memset(&ls->stats, 0, sizeof(ls->stats));

1027

1028 #ifdef HAVE_LIBMNL

1012 - ls->use_nl = false;

1029 + ls->use_mnl = false;

1030 ls->nl = NULL;

1031 ls->tmp_protocol = 0;

1032 local_sockets_libmnl_init(ls);

@@ -1072,10 +1089,10 @@ static inline void local_sockets_cleanup(LS_STATE *ls) {

1089

1090 static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) {

1091 #ifdef HAVE_LIBMNL

1075 - if(ls->nl && ls->use_nl) {

1076 - ls->use_nl = local_sockets_libmnl_get_sockets(ls, family, protocol);

1092 + if(!ls->config.no_mnl && ls->nl && ls->use_mnl) {

1093 + ls->use_mnl = local_sockets_libmnl_get_sockets(ls, family, protocol);

1094

1078 - if(ls->use_nl)

1095 + if(ls->use_mnl)

1096 return;

1097 }

1098 #endif

@@ -1083,35 +1100,64 @@ static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *fi

1100 local_sockets_read_proc_net_x(ls, filename, family, protocol);

1101 }

1102

1103 +static inline void local_sockets_track_time(LS_STATE *ls, const char *name) {

1104 + if(!ls->config.report || ls->timings_idx >= _countof(ls->timings))

1105 + return;

1106 +

1107 + usec_t now_ut = now_monotonic_usec();

1108 +

1109 + if(ls->timings_idx == 0 && !ls->timings[0].start_ut) {

1110 + ls->timings[0].start_ut = now_ut;

1111 + ls->timings[0].name = name;

1112 + }

1113 + else if(ls->timings_idx + 1 < _countof(ls->timings)) {

1114 + ls->timings[ls->timings_idx].end_ut = now_ut;

1115 + ls->timings_idx++;

1116 + ls->timings[ls->timings_idx].start_ut = now_ut;

1117 + ls->timings[ls->timings_idx].name = name;

1118 + }

1119 + else if(ls->timings_idx + 1 == _countof(ls->timings)) {

1120 + ls->timings[ls->timings_idx].end_ut = now_ut;

1121 + ls->timings_idx++; // out of bounds

1122 + }

1123 +}

1124 +

1125 static inline void local_sockets_read_all_system_sockets(LS_STATE *ls) {

1126 char path[FILENAME_MAX + 1];

1127

1128 if(ls->config.namespaces) {

1129 + local_sockets_track_time(ls, "read_namespaces");

1130 snprintfz(path, sizeof(path), "%s/proc/self/ns/net", ls->config.host_prefix);

1131 local_sockets_read_proc_inode_link(ls, path, &ls->proc_self_net_ns_inode, "net");

1132 +

1133 }

1134

1135 if(ls->config.cmdline || ls->config.comm || ls->config.pid || ls->config.namespaces) {

1136 + local_sockets_track_time(ls, "read_proc_pids");

1137 snprintfz(path, sizeof(path), "%s/proc", ls->config.host_prefix);

1138 local_sockets_find_all_sockets_in_proc(ls, path);

1139 }

1140

1141 if(ls->config.tcp4) {

1142 + local_sockets_track_time(ls, "read_tcp4");

1143 snprintfz(path, sizeof(path), "%s/proc/net/tcp", ls->config.host_prefix);

1144 local_sockets_do_family_protocol(ls, path, AF_INET, IPPROTO_TCP);

1145 }

1146

1147 if(ls->config.udp4) {

1148 + local_sockets_track_time(ls, "read_udp4");

1149 snprintfz(path, sizeof(path), "%s/proc/net/udp", ls->config.host_prefix);

1150 local_sockets_do_family_protocol(ls, path, AF_INET, IPPROTO_UDP);

1151 }

1152

1153 if(ls->config.tcp6) {

1154 + local_sockets_track_time(ls, "read_tcp6");

1155 snprintfz(path, sizeof(path), "%s/proc/net/tcp6", ls->config.host_prefix);

1156 local_sockets_do_family_protocol(ls, path, AF_INET6, IPPROTO_TCP);

1157 }

1158

1159 if(ls->config.udp6) {

1160 + local_sockets_track_time(ls, "read_udp6");

1161 snprintfz(path, sizeof(path), "%s/proc/net/udp6", ls->config.host_prefix);

1162 local_sockets_do_family_protocol(ls, path, AF_INET6, IPPROTO_UDP);

1163 }

@@ -1291,7 +1337,7 @@ struct local_sockets_namespace_worker {

1337 uint64_t inode;

1338 };

1339

1294 -static inline void *local_sockets_get_namespace_sockets(void *arg) {

1340 +static inline void *local_sockets_get_namespace_sockets_worker(void *arg) {

1341 struct local_sockets_namespace_worker *data = arg;

1342 LS_STATE *ls = data->ls;

1343 const uint64_t inode = data->inode;

@@ -1358,8 +1404,10 @@ static inline void local_sockets_namespaces(LS_STATE *ls) {

1404 workers_data[last_thread].ls = ls;

1405 workers_data[last_thread].inode = inode;

1406 workers[last_thread] = nd_thread_create(

1361 - "local-sockets-worker", NETDATA_THREAD_OPTION_JOINABLE,

1362 - local_sockets_get_namespace_sockets, &workers_data[last_thread]);

1407 + "local-sockets-worker",

1408 + NETDATA_THREAD_OPTION_JOINABLE,

1409 + local_sockets_get_namespace_sockets_worker,

1410 + &workers_data[last_thread]);

1411

1412 spinlock_lock(&ls->spinlock);

1413 }

@@ -1376,24 +1424,35 @@ static inline void local_sockets_namespaces(LS_STATE *ls) {

1424 // --------------------------------------------------------------------------------------------------------------------

1425

1426 static inline void local_sockets_process(LS_STATE *ls) {

1427 + ls->timings_idx = 0;

1428 + local_sockets_track_time(ls, "init");

1429 +

1430 // initialize our hashtables

1431 local_sockets_init(ls);

1432

1433 + local_sockets_track_time(ls, "all_sockets");

1434 +

1435 // read all sockets from /proc

1436 local_sockets_read_all_system_sockets(ls);

1437

1438 // check all socket namespaces

1386 - if(ls->config.namespaces)

1439 + if(ls->config.namespaces) {

1440 + local_sockets_track_time(ls, "switch_namespaces");

1441 local_sockets_namespaces(ls);

1442 + }

1443

1444 // detect the directions of the sockets

1390 - if(ls->config.inbound || ls->config.outbound || ls->config.local)

1445 + if(ls->config.inbound || ls->config.outbound || ls->config.local) {

1446 + local_sockets_track_time(ls, "detect_direction");

1447 local_sockets_detect_directions(ls);

1448 + }

1449

1450 // call the callback for each socket

1451 + local_sockets_track_time(ls, "output");

1452 local_sockets_foreach_local_socket_call_cb(ls);

1453

1454 // free all memory

1455 + local_sockets_track_time(ls, "cleanup");

1456 local_sockets_cleanup(ls);

1457 }

1458

src/libnetdata/os/gettid.c

+6 -1

@@ -30,4 +30,9 @@ pid_t gettid_cached(void) {

30 gettid_cached_tid = os_gettid();

32 return gettid_cached_tid;

33 -}

\ No newline at end of file

33 +}

34 +

35 +pid_t gettid_uncached(void) {

36 + gettid_cached_tid = 0;

37 + return gettid_cached();

38 +}

src/libnetdata/os/gettid.h

@@ -7,5 +7,6 @@

8 pid_t os_gettid(void);

9 pid_t gettid_cached(void);

10 +pid_t gettid_uncached(void);

12 #endif //NETDATA_GETTID_H

src/libnetdata/spawn_server/spawn_server_nofork.c

+3 -1

@@ -40,7 +40,7 @@ static int connect_to_spawn_server(const char *path, bool log) {

41 if (connect(sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) == -1) {

42 if(log)

43 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Cannot connect() to spawn server.");

43 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Cannot connect() to spawn server on path '%s'.", path);

44 close(sock);

45 return -1;

46 }

@@ -351,6 +351,8 @@ static bool spawn_server_run_callback(SPAWN_SERVER *server __maybe_unused, SPAWN

351 }

352

353 pid_t pid = fork();

354 + gettid_uncached(); // make sure the logger logs valid pids

355 +

356 if (pid < 0) {

357 // fork failed

358