@cryptotaxi247 / netdata-1 / commits / 166e9caff

spawn server (Windows support for external plugins) (#17866)

* listening ipv6 sockets may be both ipv4 and ipv6, depending on the IPV6_ONLY flag * working libmnl ipv46 detection and added latency and retransmits from TCP_INFO * fix aggregations for rtt and retrans * code cleanup * code cleanup * code cleanup again * restore field renames * count namespaces * run namespaces in parallel * add libmnl to buildinfo * lock around safe_fork() * libmnl ports are in network byte order * posix spawn server for both executables and callback functions * local-sockets and network-viewer using the new spawn server * cleanup spawn servers sockets * spawn server stdin and stdout are linked to /dev/null * no need for spinlock in spawn server * empty all parameters * new spawn server is now used for plugins.d plugins * fix for environ * claiming script runs via the new spawn server * tc.plugin uses the new spawn server * analytics, buildinfo and cgroups.plugin use the new spawn server * cgroup-discovery uses the new spawn server * added ability to wait or kill spawned processes * removed old spawn server and now alert notifications use the new one * remove left-overs * hide spawn server internals; started working on windows version of the spawn server * fixes for windows * more windows work * more work on windows * added debug log to spawn server * fix compilation warnings * enable static threads on windows * running external plugins * working spawn server on windows * spawn server logs to collectoers.log * log windows last error together with errno * log updates * cleanup * decode_argv does not add an empty parameter * removed debug log * removed debug return * rework on close_range() * eliminate the need for waitid() * clear errno on the signal handler * added universal os_setproctitle() call to support FreeBSD too * os_get_pid_max() for windows and macos * isolate pids array from the rest of the code in apps.plugin so that it can be turned to a hashtable

Costa Tsaousis committed Jul 10, 2024 at 14:23 UTC 166e9caffaf2ace7312fd02a2cfeef6e7ab09ea2

83 files changed +2778 -2455

CMakeLists.txt

+8 -12

@@ -668,8 +668,6 @@ set(LIBNETDATA_FILES

668 src/libnetdata/os/byteorder.h

669 src/libnetdata/onewayalloc/onewayalloc.c

670 src/libnetdata/onewayalloc/onewayalloc.h

671 - src/libnetdata/popen/popen.c

672 - src/libnetdata/popen/popen.h

671 src/libnetdata/procfile/procfile.c

672 src/libnetdata/procfile/procfile.h

673 src/libnetdata/query_progress/progress.c

@@ -720,8 +718,6 @@ set(LIBNETDATA_FILES

718 src/libnetdata/linked-lists.h

719 src/libnetdata/storage-point.h

720 src/libnetdata/bitmap64.h

723 - src/libnetdata/os/waitid.c

724 - src/libnetdata/os/waitid.h

721 src/libnetdata/os/gettid.c

722 src/libnetdata/os/gettid.h

723 src/libnetdata/os/adjtimex.c

@@ -750,6 +746,14 @@ set(LIBNETDATA_FILES

746 src/libnetdata/os/setenv.h

747 src/libnetdata/os/strndup.c

748 src/libnetdata/os/strndup.h

749 + src/libnetdata/spawn_server/spawn_server.c

750 + src/libnetdata/spawn_server/spawn_server.h

751 + src/libnetdata/spawn_server/spawn_popen.c

752 + src/libnetdata/spawn_server/spawn_popen.h

753 + src/libnetdata/os/close_range.c

754 + src/libnetdata/os/close_range.h

755 + src/libnetdata/os/setproctitle.c

756 + src/libnetdata/os/setproctitle.h

757 )

758

759 if(ENABLE_PLUGIN_EBPF)

@@ -1216,13 +1220,6 @@ set(CLAIM_PLUGIN_FILES

1220 src/claim/claim.h

1221 )

1222

1219 -set(SPAWN_PLUGIN_FILES

1220 - src/spawn/spawn.c

1221 - src/spawn/spawn_server.c

1222 - src/spawn/spawn_client.c

1223 - src/spawn/spawn.h

1224 -)

1225 -

1223 set(ACLK_ALWAYS_BUILD

1224 src/aclk/aclk_rrdhost_state.h

1225 src/aclk/aclk_proxy.c

@@ -1410,7 +1407,6 @@ set(NETDATA_FILES

1407 ${STREAMING_PLUGIN_FILES}

1408 ${WEB_PLUGIN_FILES}

1409 ${CLAIM_PLUGIN_FILES}

1413 - ${SPAWN_PLUGIN_FILES}

1410 ${ACLK_ALWAYS_BUILD}

1411 ${PROFILE_PLUGIN_FILES}

1412 )

packaging/utils/compile-on-windows.sh

+7 -1

@@ -26,6 +26,12 @@ then

26 exit 0

27 fi

29 +BUILD_FOR_PACKAGING="Off"

30 +if [ "${1}" = "package" ]

31 +then

32 + BUILD_FOR_PACKAGING="On"

33 +fi

34 +

35 export PATH="/usr/local/bin:${PATH}"

37 WT_ROOT="$(pwd)"

@@ -54,7 +60,7 @@ fi

60 -DCMAKE_INSTALL_PREFIX="/opt/netdata" \

61 -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \

62 -DCMAKE_C_FLAGS="-fstack-protector-all -O0 -ggdb -Wall -Wextra -Wno-char-subscripts -Wa,-mbig-obj -pipe -DNETDATA_INTERNAL_CHECKS=1 -D_FILE_OFFSET_BITS=64 -D__USE_MINGW_ANSI_STDIO=1" \

57 - -DBUILD_FOR_PACKAGING=On \

63 + -DBUILD_FOR_PACKAGING=${BUILD_FOR_PACKAGING} \

64 -DUSE_MOLD=Off \

65 -DNETDATA_USER="${USER}" \

66 -DDEFAULT_FEATURE_STATE=Off \

src/aclk/aclk_rx_msgs.c

+3 -3

@@ -106,13 +106,13 @@ static inline int aclk_v2_payload_get_query(const char *payload, char **query_ur

106 else if(strncmp(payload, "DELETE /", 8) == 0)

107 start = payload + 7;

108 else {

109 - errno = 0;

109 + errno_clear();

110 netdata_log_error("Only accepting requests that start with GET, POST, PUT, DELETE from CLOUD.");

111 return 1;

112 }

113

114 if(!(end = strstr(payload, HTTP_1_1 HTTP_ENDL))) {

115 - errno = 0;

115 + errno_clear();

116 netdata_log_error("Doesn't look like HTTP GET request.");

117 return 1;

118 }

@@ -127,7 +127,7 @@ static int aclk_handle_cloud_http_request_v2(struct aclk_request *cloud_to_agent

127 {

128 aclk_query_t query;

129

130 - errno = 0;

130 + errno_clear();

131 if (cloud_to_agent->version < ACLK_V_COMPRESSION) {

132 netdata_log_error(

133 "This handler cannot reply to request with version older than %d, received %d.",

src/claim/claim.c

+6 -9

@@ -53,11 +53,8 @@ CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, con

53 }

55 #ifndef DISABLE_CLOUD

56 - int exit_code;

57 - pid_t command_pid;

56 char command_exec_buffer[CLAIMING_COMMAND_LENGTH + 1];

57 char command_line_buffer[CLAIMING_COMMAND_LENGTH + 1];

60 - FILE *fp_child_output, *fp_child_input;

59 // This is guaranteed to be set early in main via post_conf_load()

60 char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);

@@ -92,17 +89,17 @@ CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, con

89 claiming_arguments);

91 netdata_log_info("Executing agent claiming command: %s", command_exec_buffer);

95 - fp_child_output = netdata_popen(command_line_buffer, &command_pid, &fp_child_input);

96 - if(!fp_child_output) {

92 + POPEN_INSTANCE *instance = spawn_popen_run(command_exec_buffer);

93 + if(!instance) {

94 netdata_log_error("Cannot popen(\"%s\").", command_exec_buffer);

95 return CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT;

96 }

98 netdata_log_info("Waiting for claiming command '%s' to finish.", command_exec_buffer);

99 char read_buffer[100 + 1];

103 - while (fgets(read_buffer, 100, fp_child_output) != NULL) ;

100 + while (fgets(read_buffer, 100, instance->child_stdout_fp) != NULL) ;

101

105 - exit_code = netdata_pclose(fp_child_input, fp_child_output, command_pid);

102 + int exit_code = spawn_popen_wait(instance);

103

104 netdata_log_info("Agent claiming command '%s' returned with code %d", command_exec_buffer, exit_code);

105 if (0 == exit_code) {

@@ -113,7 +110,7 @@ CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, con

110 netdata_log_error("Agent claiming command '%s' failed to complete its run", command_exec_buffer);

111 return CLAIM_AGENT_CLAIM_SCRIPT_FAILED;

112 }

116 - errno = 0;

113 + errno_clear();

114 unsigned maximum_known_exit_code = sizeof(claiming_errors) / sizeof(claiming_errors[0]) - 1;

115

116 if ((unsigned)exit_code > maximum_known_exit_code) {

@@ -214,7 +211,7 @@ void load_cloud_conf(int silent)

211 netdata_cloud_enabled = CONFIG_BOOLEAN_NO;

212

213 char *filename;

217 - errno = 0;

214 + errno_clear();

215

216 int ret = 0;

217

src/collectors/apps.plugin/apps_plugin.c

+11 -28

@@ -51,7 +51,6 @@ size_t

51 inodes_changed_counter = 0,

52 links_changed_counter = 0,

53 targets_assignment_counter = 0,

54 - all_pids_count = 0, // the number of processes running

54 apps_groups_targets_count = 0; // # of apps_groups.conf targets

56 int

@@ -136,20 +135,6 @@ struct target

135

136 size_t pagesize;

137

139 -struct pid_stat

140 - *root_of_pids = NULL, // global list of all processes running

141 - **all_pids = NULL; // to avoid allocations, we pre-allocate

142 - // a pointer for each pid in the entire pid space.

143 -

144 -#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)

145 -// Another pre-allocated list of all possible pids.

146 -// We need it to pids and assign them a unique sortlist id, so that we

147 -// read parents before children. This is needed to prevent a situation where

148 -// a child is found running, but until we read its parent, it has exited and

149 -// its parent has accumulated its resources.

150 -pid_t *all_pids_sortlist = NULL;

151 -#endif

152 -

138 // ----------------------------------------------------------------------------

139

140 int managed_log(struct pid_stat *p, PID_LOG log, int status) {

@@ -208,7 +193,7 @@ int managed_log(struct pid_stat *p, PID_LOG log, int status) {

193 }

194 }

195 }

211 - errno = 0;

196 + errno_clear();

197 }

198 else if(unlikely(p->log_thrown & log)) {

199 // netdata_log_error("unsetting log %u on pid %d", log, p->pid);

@@ -300,12 +285,14 @@ static void apply_apps_groups_targets_inheritance(void) {

285 }

286

287 // init goes always to default target

303 - if(all_pids[INIT_PID] && !all_pids[INIT_PID]->matched_by_config)

304 - all_pids[INIT_PID]->target = apps_groups_default_target;

288 + struct pid_stat *pi = find_pid_entry(INIT_PID);

289 + if(pi && !pi->matched_by_config)

290 + pi->target = apps_groups_default_target;

291

292 // pid 0 goes always to default target

307 - if(all_pids[0] && !all_pids[INIT_PID]->matched_by_config)

308 - all_pids[0]->target = apps_groups_default_target;

293 + pi = find_pid_entry(0);

294 + if(pi && !pi->matched_by_config)

295 + pi->target = apps_groups_default_target;

296

297 // give a default target on all top level processes

298 if(unlikely(debug_enabled)) loops++;

@@ -320,8 +307,9 @@ static void apply_apps_groups_targets_inheritance(void) {

307 p->sortlist = sortlist++;

308 }

309

323 - if(all_pids[1])

324 - all_pids[1]->sortlist = sortlist++;

310 + pi = find_pid_entry(1);

311 + if(pi)

312 + pi->sortlist = sortlist++;

313

314 // give a target to all merged child processes

315 found = 1;

@@ -1052,12 +1040,7 @@ int main(int argc, char **argv) {

1040 netdata_log_info("started on pid %d", getpid());

1041

1042 users_and_groups_init();

1055 -

1056 -#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)

1057 - all_pids_sortlist = callocz(sizeof(pid_t), (size_t)pid_max + 1);

1058 -#endif

1059 -

1060 - all_pids = callocz(sizeof(struct pid_stat *), (size_t) pid_max + 1);

1043 + pids_init();

1044

1045 // ------------------------------------------------------------------------

1046 // the event loop for functions

src/collectors/apps.plugin/apps_plugin.h

+4 -6

@@ -17,9 +17,7 @@

17 #include <sys/proc_info.h>

18 #include <sys/sysctl.h>

19 #include <mach/mach_time.h> // For mach_timebase_info_data_t and mach_timebase_info

20 -#endif

22 -#if defined(__APPLE__)

21 extern mach_timebase_info_data_t mach_info;

22 #endif

@@ -47,7 +45,6 @@ struct pid_info {

45 struct proc_taskinfo taskinfo;

46 struct proc_bsdinfo bsdinfo;

47 struct rusage_info_v4 rusageinfo;

50 -

48 };

49 #endif

@@ -467,9 +464,7 @@ extern struct target

464 *users_root_target,

465 *groups_root_target;

466

470 -extern struct pid_stat

471 - *root_of_pids,

472 - **all_pids;

467 +extern struct pid_stat *root_of_pids;

468

469 extern int update_every;

470 extern unsigned int time_factor;

@@ -559,4 +554,7 @@ void send_charts_updates_to_netdata(struct target *root, const char *type, const

554 void send_collected_data_to_netdata(struct target *root, const char *type, usec_t dt);

555 void send_resource_usage_to_netdata(usec_t dt);

556

557 +void pids_init(void);

558 +struct pid_stat *find_pid_entry(pid_t pid);

559 +

560 #endif //NETDATA_APPS_PLUGIN_H

src/collectors/apps.plugin/apps_proc_pid_limits.c

+1 -1

@@ -33,7 +33,7 @@ static inline bool read_proc_pid_limits_per_os(struct pid_stat *p, void *ptr __m

33 bool ret = false;

34 bool read_limits = false;

36 - errno = 0;

36 + errno_clear();

37 proc_pid_limits_buffer[0] = '\0';

39 kernel_uint_t all_fds = pid_openfds_sum(p);

src/collectors/apps.plugin/apps_proc_pids.c

+33 -7

@@ -2,18 +2,44 @@

3 #include "apps_plugin.h"

5 -static inline struct pid_stat *get_pid_entry(pid_t pid) {

6 - if(likely(all_pids[pid]))

7 - return all_pids[pid];

5 +static struct pid_stat **all_pids = NULL;

6 +size_t all_pids_count = 0; // the number of processes running

7 +

8 +struct pid_stat *root_of_pids = NULL; // global linked list of all processes running

9 +

10 +#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)

11 +// Another pre-allocated list of all possible pids.

12 +// We need it to assign them a unique sortlist id, so that we

13 +// read parents before children. This is needed to prevent a situation where

14 +// a child is found running, but until we read its parent, it has exited and

15 +// its parent has accumulated its resources.

16 +pid_t *all_pids_sortlist = NULL;

17 +#endif

18 +

19 +void pids_init(void) {

20 +#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)

21 + all_pids_sortlist = callocz(sizeof(pid_t), (size_t)pid_max + 1);

22 +#endif

23 +

24 + all_pids = callocz(sizeof(struct pid_stat *), (size_t) pid_max + 1);

25 +}

9 - struct pid_stat *p = callocz(sizeof(struct pid_stat), 1);

27 +inline struct pid_stat *find_pid_entry(pid_t pid) {

28 + return all_pids[pid];

29 +}

30 +

31 +static inline struct pid_stat *get_or_allocate_pid_entry(pid_t pid) {

32 + struct pid_stat *p = find_pid_entry(pid);

33 + if(likely(p))

34 + return p;

35 +

36 + p = callocz(sizeof(struct pid_stat), 1);

37 p->fds = mallocz(sizeof(struct pid_fd) * MAX_SPARE_FDS);

38 p->fds_size = MAX_SPARE_FDS;

39 init_pid_fds(p, 0, p->fds_size);

40 p->pid = pid;

42 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(root_of_pids, p, prev, next);

16 -

43 all_pids[pid] = p;

44 all_pids_count++;

@@ -21,7 +47,7 @@ static inline struct pid_stat *get_pid_entry(pid_t pid) {

47 }

49 static inline void del_pid_entry(pid_t pid) {

24 - struct pid_stat *p = all_pids[pid];

50 + struct pid_stat *p = find_pid_entry(pid);

52 if(unlikely(!p)) {

53 netdata_log_error("attempted to free pid %d that is not allocated.", pid);

@@ -62,7 +88,7 @@ static inline int collect_data_for_pid(pid_t pid, void *ptr) {

88 return 0;

89 }

65 - struct pid_stat *p = get_pid_entry(pid);

91 + struct pid_stat *p = get_or_allocate_pid_entry(pid);

92 if(unlikely(!p || p->read)) return 0;

93 p->read = true;

src/collectors/cgroups.plugin/cgroup-discovery.c

+8 -12

@@ -178,11 +178,9 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) {

178

179 netdata_log_debug(D_CGROUP, "looking for the name of cgroup '%s' with chart id '%s'", cg->id, cg->chart_id);

180 netdata_log_debug(D_CGROUP, "executing command %s \"%s\" for cgroup '%s'", cgroups_rename_script, cg->intermediate_id, cg->chart_id);

181 - pid_t cgroup_pid;

181

183 - FILE *fp_child_input, *fp_child_output;

184 - (void)netdata_popen_raw_default_flags_and_environment(&cgroup_pid, &fp_child_input, &fp_child_output, cgroups_rename_script, cg->id, cg->intermediate_id);

185 - if (!fp_child_output) {

182 + POPEN_INSTANCE *instance = spawn_popen_run_variadic(cgroups_rename_script, cg->id, cg->intermediate_id, NULL);

183 + if (!instance) {

184 collector_error("CGROUP: cannot popen(%s \"%s\", \"r\").", cgroups_rename_script, cg->intermediate_id);

185 cg->pending_renames = 0;

186 cg->processed = 1;

@@ -190,8 +188,8 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) {

188 }

189

190 char buffer[CGROUP_CHARTID_LINE_MAX + 1];

193 - char *new_name = fgets(buffer, CGROUP_CHARTID_LINE_MAX, fp_child_output);

194 - int exit_code = netdata_pclose(fp_child_input, fp_child_output, cgroup_pid);

191 + char *new_name = fgets(buffer, CGROUP_CHARTID_LINE_MAX, instance->child_stdout_fp);

192 + int exit_code = spawn_popen_wait(instance);

193

194 switch (exit_code) {

195 case 0:

@@ -1085,7 +1083,6 @@ static void cgroup_cleanup_ebpf_integration()

1083 static inline void read_cgroup_network_interfaces(struct cgroup *cg) {

1084 netdata_log_debug(D_CGROUP, "looking for the network interfaces of cgroup '%s' with chart id '%s'", cg->id, cg->chart_id);

1085

1088 - pid_t cgroup_pid;

1086 char cgroup_identifier[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];

1087

1088 if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {

@@ -1096,16 +1093,15 @@ static inline void read_cgroup_network_interfaces(struct cgroup *cg) {

1093 }

1094

1095 netdata_log_debug(D_CGROUP, "executing cgroup_identifier %s --cgroup '%s' for cgroup '%s'", cgroups_network_interface_script, cgroup_identifier, cg->id);

1099 - FILE *fp_child_input, *fp_child_output;

1100 - (void)netdata_popen_raw_default_flags_and_environment(&cgroup_pid, &fp_child_input, &fp_child_output, cgroups_network_interface_script, "--cgroup", cgroup_identifier);

1101 - if(!fp_child_output) {

1096 + POPEN_INSTANCE *instance = spawn_popen_run_variadic(cgroups_network_interface_script, "--cgroup", cgroup_identifier, NULL);

1097 + if(!instance) {

1098 collector_error("CGROUP: cannot popen(%s --cgroup \"%s\", \"r\").", cgroups_network_interface_script, cgroup_identifier);

1099 return;

1100 }

1101

1102 char *s;

1103 char buffer[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];

1108 - while((s = fgets(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, fp_child_output))) {

1104 + while((s = fgets(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, instance->child_stdout_fp))) {

1105 trim(s);

1106

1107 if(*s && *s != '\n') {

@@ -1145,7 +1141,7 @@ static inline void read_cgroup_network_interfaces(struct cgroup *cg) {

1141 }

1142 }

1143

1148 - netdata_pclose(fp_child_input, fp_child_output, cgroup_pid);

1144 + spawn_popen_wait(instance);

1145 }

1146

1147 static inline void discovery_process_cgroup(struct cgroup *cg) {

src/collectors/cgroups.plugin/cgroup-network.c

+14 -16

@@ -421,19 +421,19 @@ void detect_veth_interfaces(pid_t pid) {

421

422 host = read_proc_net_dev("host", netdata_configured_host_prefix);

423 if(!host) {

424 - errno = 0;

424 + errno_clear();

425 collector_error("cannot read host interface list.");

426 goto cleanup;

427 }

428

429 if(!eligible_ifaces(host)) {

430 - errno = 0;

430 + errno_clear();

431 collector_info("there are no double-linked host interfaces available.");

432 goto cleanup;

433 }

434

435 if(switch_namespace(netdata_configured_host_prefix, pid)) {

436 - errno = 0;

436 + errno_clear();

437 collector_error("cannot switch to the namespace of pid %u", (unsigned int) pid);

438 goto cleanup;

439 }

@@ -444,13 +444,13 @@ void detect_veth_interfaces(pid_t pid) {

444

445 cgroup = read_proc_net_dev("cgroup", NULL);

446 if(!cgroup) {

447 - errno = 0;

447 + errno_clear();

448 collector_error("cannot read cgroup interface list.");

449 goto cleanup;

450 }

451

452 if(!eligible_ifaces(cgroup)) {

453 - errno = 0;

453 + errno_clear();

454 collector_error("there are not double-linked cgroup interfaces available.");

455 goto cleanup;

456 }

@@ -505,22 +505,20 @@ void call_the_helper(pid_t pid, const char *cgroup) {

505

506 collector_info("running: %s", command);

507

508 - pid_t cgroup_pid;

509 - FILE *fp_child_input, *fp_child_output;

508 + POPEN_INSTANCE *pi;

509

511 - if(cgroup) {

512 - (void)netdata_popen_raw_default_flags(&cgroup_pid, environment, &fp_child_input, &fp_child_output, PLUGINS_DIR "/cgroup-network-helper.sh", "--cgroup", cgroup);

513 - }

510 + if(cgroup)

511 + pi = spawn_popen_run_variadic(PLUGINS_DIR "/cgroup-network-helper.sh", "--cgroup", cgroup, NULL);

512 else {

513 char buffer[100];

514 snprintfz(buffer, sizeof(buffer) - 1, "%d", pid);

517 - (void)netdata_popen_raw_default_flags(&cgroup_pid, environment, &fp_child_input, &fp_child_output, PLUGINS_DIR "/cgroup-network-helper.sh", "--pid", buffer);

515 + pi = spawn_popen_run_variadic(PLUGINS_DIR "/cgroup-network-helper.sh", "--pid", buffer, NULL);

516 }

517

520 - if(fp_child_output) {

518 + if(pi) {

519 char buffer[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];

520 char *s;

523 - while((s = fgets(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, fp_child_output))) {

521 + while((s = fgets(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, pi->child_stdout_fp))) {

522 trim(s);

523

524 if(*s && *s != '\n') {

@@ -536,7 +534,7 @@ void call_the_helper(pid_t pid, const char *cgroup) {

534 }

535 }

536

539 - netdata_pclose(fp_child_input, fp_child_output, cgroup_pid);

537 + spawn_popen_kill(pi);

538 }

539 else

540 collector_error("cannot execute cgroup-network helper script: %s", command);

@@ -701,7 +699,7 @@ int main(int argc, char **argv) {

699 pid = atoi(argv[arg+1]);

700

701 if(pid <= 0) {

704 - errno = 0;

702 + errno_clear();

703 collector_error("Invalid pid %d given", (int) pid);

704 return 2;

705 }

@@ -719,7 +717,7 @@ int main(int argc, char **argv) {

717 if(helper) call_the_helper(pid, cgroup);

718

719 if(pid <= 0 && !detected_devices) {

722 - errno = 0;

720 + errno_clear();

721 collector_error("Cannot find a cgroup PID from cgroup '%s'", cgroup);

722 }

723 }

src/collectors/cgroups.plugin/sys_fs_cgroup.c

+10 -23

@@ -73,30 +73,19 @@ struct discovery_thread discovery_thread;

73 #define MAXSIZE_PROC_CMDLINE 4096

74 static enum cgroups_systemd_setting cgroups_detect_systemd(const char *exec)

75 {

76 - pid_t command_pid;

76 enum cgroups_systemd_setting retval = SYSTEMD_CGROUP_ERR;

77 char buf[MAXSIZE_PROC_CMDLINE];

78 char *begin, *end;

81 - FILE *fp_child_input;

82 - FILE *fp_child_output = netdata_popen(exec, &command_pid, &fp_child_input);

83 -

84 - if (!fp_child_output)

80 + POPEN_INSTANCE *pi = spawn_popen_run(exec);

81 + if(!pi)

82 return retval;

87 - int fd = fileno(fp_child_output);

88 - if (fd == -1 ) {

89 - collector_error("Cannot get the output of \"%s\": failed to get file descriptor", exec);

90 - netdata_pclose(fp_child_input, fp_child_output, command_pid);

91 - return retval;

92 - }

93 -

84 struct pollfd pfd;

95 - pfd.fd = fd;

85 + pfd.fd = spawn_server_instance_read_fd(pi->si);

86 pfd.events = POLLIN;

88 int timeout = 3000; // milliseconds

99 -

89 int ret = poll(&pfd, 1, timeout);

91 if (ret == -1) {

@@ -104,7 +93,7 @@ static enum cgroups_systemd_setting cgroups_detect_systemd(const char *exec)

93 } else if (ret == 0) {

94 collector_info("Cannot get the output of \"%s\" within timeout (%d ms)", exec, timeout);

95 } else {

107 - while (fgets(buf, MAXSIZE_PROC_CMDLINE, fp_child_output) != NULL) {

96 + while (fgets(buf, MAXSIZE_PROC_CMDLINE, pi->child_stdout_fp) != NULL) {

97 if ((begin = strstr(buf, SYSTEMD_HIERARCHY_STRING))) {

98 end = begin = begin + strlen(SYSTEMD_HIERARCHY_STRING);

99 if (!*begin)

@@ -123,7 +112,7 @@ static enum cgroups_systemd_setting cgroups_detect_systemd(const char *exec)

112 }

113 }

114

126 - if (netdata_pclose(fp_child_input, fp_child_output, command_pid))

115 + if(spawn_popen_wait(pi) != 0)

116 return SYSTEMD_CGROUP_ERR;

117

118 return retval;

@@ -159,25 +148,23 @@ static enum cgroups_type cgroups_try_detect_version()

148

149 collector_info("cgroups version: can't detect using statfs (fs type), falling back to heuristics.");

150

162 - pid_t command_pid;

151 char buf[MAXSIZE_PROC_CMDLINE];

152 enum cgroups_systemd_setting systemd_setting;

153 int cgroups2_available = 0;

154

155 // 1. check if cgroups2 available on system at all

168 - FILE *fp_child_input;

169 - FILE *fp_child_output = netdata_popen("grep cgroup /proc/filesystems", &command_pid, &fp_child_input);

170 - if (!fp_child_output) {

171 - collector_error("popen failed");

156 + POPEN_INSTANCE *instance = spawn_popen_run("grep cgroup /proc/filesystems");

157 + if(!instance) {

158 + collector_error("cannot run 'grep cgroup /proc/filesystems'");

159 return CGROUPS_AUTODETECT_FAIL;

160 }

174 - while (fgets(buf, MAXSIZE_PROC_CMDLINE, fp_child_output) != NULL) {

161 + while (fgets(buf, MAXSIZE_PROC_CMDLINE, instance->child_stdout_fp) != NULL) {

162 if (strstr(buf, "cgroup2")) {

163 cgroups2_available = 1;

164 break;

165 }

166 }

180 - if(netdata_pclose(fp_child_input, fp_child_output, command_pid))

167 + if(spawn_popen_wait(instance) != 0)

168 return CGROUPS_AUTODETECT_FAIL;

169

170 if(!cgroups2_available)

src/collectors/cups.plugin/cups_plugin.c

+1 -1

@@ -231,7 +231,7 @@ int main(int argc, char **argv) {

231

232 parse_command_line(argc, argv);

233

234 - errno = 0;

234 + errno_clear();

235

236 dict_dest_job_metrics = dictionary_create(DICT_OPTION_SINGLE_THREADED);

237

src/collectors/ebpf.plugin/ebpf_apps.c

+1 -1

@@ -441,7 +441,7 @@ static inline int managed_log(struct ebpf_pid_stat *p, uint32_t log, int status)

441 }

442 }

443 }

444 - errno = 0;

444 + errno_clear();

445 } else if (unlikely(p->log_thrown & log)) {

446 // netdata_log_error("unsetting log %u on pid %d", log, p->pid);

447 p->log_thrown &= ~log;

src/collectors/freeipmi.plugin/freeipmi_plugin.c

+2 -2

@@ -1120,7 +1120,7 @@ static void netdata_update_ipmi_sel_events_count(struct netdata_ipmi_state *stt,

1120 }

1121

1122 int netdata_ipmi_collect_data(struct ipmi_monitoring_ipmi_config *ipmi_config, IPMI_COLLECTION_TYPE type, struct netdata_ipmi_state *stt) {

1123 - errno = 0;

1123 + errno_clear();

1124

1125 if(type & IPMI_COLLECT_TYPE_SENSORS) {

1126 stt->sensors.collected = 0;

@@ -1930,7 +1930,7 @@ int main (int argc, char **argv) {

1930 collector_error("%s(): ignoring parameter '%s'", __FUNCTION__, argv[i]);

1931 }

1932

1933 - errno = 0;

1933 + errno_clear();

1934

1935 if(freq_s && freq_s < update_every)

1936 collector_info("%s(): update frequency %d seconds is too small for IPMI. Using %d.",

src/collectors/network-viewer.plugin/network-viewer.c

+265 -59

@@ -4,12 +4,22 @@

4 #include "libnetdata/libnetdata.h"

5 #include "libnetdata/required_dummies.h"

7 +static SPAWN_SERVER *spawn_srv = NULL;

8 +

9 #define ENABLE_DETAILED_VIEW

11 #define LOCAL_SOCKETS_EXTENDED_MEMBERS struct { \

12 size_t count; \

11 - const char *local_address_space; \

12 - const char *remote_address_space; \

13 + struct { \

14 + pid_t pid; \

15 + uid_t uid; \

16 + SOCKET_DIRECTION direction; \

17 + int state; \

18 + uint64_t net_ns_inode; \

19 + struct socket_endpoint server; \

20 + const char *local_address_space; \

21 + const char *remote_address_space; \

22 + } aggregated_key; \

23 } network_viewer;

25 #include "libnetdata/maps/local-sockets.h"

@@ -57,19 +67,49 @@ ENUM_STR_MAP_DEFINE(TCP_STATE) = {

67 };

68 ENUM_STR_DEFINE_FUNCTIONS(TCP_STATE, 0, "unknown");

60 -static void local_socket_to_json_array(BUFFER *wb, LOCAL_SOCKET *n, uint64_t proc_self_net_ns_inode, bool aggregated) {

70 +struct sockets_stats {

71 + BUFFER *wb;

72 +

73 + struct {

74 + uint32_t tcpi_rtt;

75 + uint32_t tcpi_rcv_rtt;

76 + uint32_t tcpi_total_retrans;

77 + } max;

78 +};

79 +

80 +static void local_socket_to_json_array(struct sockets_stats *st, LOCAL_SOCKET *n, uint64_t proc_self_net_ns_inode, bool aggregated) {

81 + if(n->direction == SOCKET_DIRECTION_NONE)

82 + return;

83 +

84 + BUFFER *wb = st->wb;

85 +

86 char local_address[INET6_ADDRSTRLEN];

87 char remote_address[INET6_ADDRSTRLEN];

88 char *protocol;

90 if(n->local.family == AF_INET) {

91 ipv4_address_to_txt(n->local.ip.ipv4, local_address);

67 - ipv4_address_to_txt(n->remote.ip.ipv4, remote_address);

92 +

93 + if(local_sockets_is_zero_address(&n->remote))

94 + remote_address[0] = '\0';

95 + else

96 + ipv4_address_to_txt(n->remote.ip.ipv4, remote_address);

97 +

98 protocol = n->local.protocol == IPPROTO_TCP ? "tcp4" : "udp4";

99 }

100 + else if(is_local_socket_ipv46(n)) {

101 + strncpyz(local_address, "*", sizeof(local_address) - 1);

102 + remote_address[0] = '\0';

103 + protocol = n->local.protocol == IPPROTO_TCP ? "tcp46" : "udp46";

104 + }

105 else if(n->local.family == AF_INET6) {

106 ipv6_address_to_txt(&n->local.ip.ipv6, local_address);

72 - ipv6_address_to_txt(&n->remote.ip.ipv6, remote_address);

107 +

108 + if(local_sockets_is_zero_address(&n->remote))

109 + remote_address[0] = '\0';

110 + else

111 + ipv6_address_to_txt(&n->remote.ip.ipv6, remote_address);

112 +

113 protocol = n->local.protocol == IPPROTO_TCP ? "tcp6" : "udp6";

114 }

115 else

@@ -117,43 +157,45 @@ static void local_socket_to_json_array(BUFFER *wb, LOCAL_SOCKET *n, uint64_t pro

157 buffer_json_add_array_item_string(wb, local_address);

158 buffer_json_add_array_item_uint64(wb, n->local.port);

159 }

120 - buffer_json_add_array_item_string(wb, n->network_viewer.local_address_space);

160 + buffer_json_add_array_item_string(wb, n->network_viewer.aggregated_key.local_address_space);

161

162 if(!aggregated) {

163 buffer_json_add_array_item_string(wb, remote_address);

164 buffer_json_add_array_item_uint64(wb, n->remote.port);

165 }

126 - buffer_json_add_array_item_string(wb, n->network_viewer.remote_address_space);

166 + buffer_json_add_array_item_string(wb, n->network_viewer.aggregated_key.remote_address_space);

167

128 - uint16_t server_port = 0;

129 - const char *server_address = NULL;

130 - const char *client_address_space = NULL;

131 - const char *server_address_space = NULL;

168 + const char *server_address;

169 + const char *client_address_space;

170 + const char *server_address_space;

171 switch (n->direction) {

172 case SOCKET_DIRECTION_LISTEN:

173 case SOCKET_DIRECTION_INBOUND:

174 case SOCKET_DIRECTION_LOCAL_INBOUND:

136 - server_port = n->local.port;

175 server_address = local_address;

138 - server_address_space = n->network_viewer.local_address_space;

139 - client_address_space = n->network_viewer.remote_address_space;

176 + server_address_space = n->network_viewer.aggregated_key.local_address_space;

177 + client_address_space = n->network_viewer.aggregated_key.remote_address_space;

178 break;

179

180 case SOCKET_DIRECTION_OUTBOUND:

181 case SOCKET_DIRECTION_LOCAL_OUTBOUND:

144 - server_port = n->remote.port;

182 server_address = remote_address;

146 - server_address_space = n->network_viewer.remote_address_space;

147 - client_address_space = n->network_viewer.local_address_space;

183 + server_address_space = n->network_viewer.aggregated_key.remote_address_space;

184 + client_address_space = n->network_viewer.aggregated_key.local_address_space;

185 break;

186

187 case SOCKET_DIRECTION_NONE:

188 + server_address = NULL;

189 + client_address_space = NULL;

190 + server_address_space = NULL;

191 break;

192 }

153 - if(aggregated)

193 +

194 + if(aggregated) {

195 buffer_json_add_array_item_string(wb, server_address);

196 + }

197

156 - buffer_json_add_array_item_uint64(wb, server_port);

198 + buffer_json_add_array_item_uint64(wb, n->network_viewer.aggregated_key.server.port);

199

200 if(aggregated) {

201 buffer_json_add_array_item_string(wb, client_address_space);

@@ -162,58 +204,176 @@ static void local_socket_to_json_array(BUFFER *wb, LOCAL_SOCKET *n, uint64_t pro

204

205 // buffer_json_add_array_item_uint64(wb, n->inode);

206 // buffer_json_add_array_item_uint64(wb, n->net_ns_inode);

207 +

208 + // RTT

209 + buffer_json_add_array_item_double(wb, (double)n->info.tcp.tcpi_rtt / (double)USEC_PER_MS);

210 + if(st->max.tcpi_rtt < n->info.tcp.tcpi_rtt)

211 + st->max.tcpi_rtt = n->info.tcp.tcpi_rtt;

212 +

213 + // Receiver RTT

214 + buffer_json_add_array_item_double(wb, (double)n->info.tcp.tcpi_rcv_rtt / (double)USEC_PER_MS);

215 + if(st->max.tcpi_rcv_rtt < n->info.tcp.tcpi_rcv_rtt)

216 + st->max.tcpi_rcv_rtt = n->info.tcp.tcpi_rcv_rtt;

217 +

218 + // Retransmissions

219 + buffer_json_add_array_item_uint64(wb, n->info.tcp.tcpi_total_retrans);

220 + if(st->max.tcpi_total_retrans < n->info.tcp.tcpi_total_retrans)

221 + st->max.tcpi_total_retrans = n->info.tcp.tcpi_total_retrans;

222 +

223 + // count

224 buffer_json_add_array_item_uint64(wb, n->network_viewer.count);

225 }

226 buffer_json_array_close(wb);

227 }

228

170 -static void local_sockets_cb_to_json(LS_STATE *ls, LOCAL_SOCKET *n, void *data) {

229 +static void populate_aggregated_key(LOCAL_SOCKET *n) {

230 n->network_viewer.count = 1;

172 - n->network_viewer.local_address_space = local_sockets_address_space(&n->local);

173 - n->network_viewer.remote_address_space = local_sockets_address_space(&n->remote);

174 - local_socket_to_json_array(data, n, ls->proc_self_net_ns_inode, false);

175 -}

231

177 -static void local_sockets_cb_to_aggregation(LS_STATE *ls __maybe_unused, LOCAL_SOCKET *n, void *data) {

178 - SIMPLE_HASHTABLE_AGGREGATED_SOCKETS *ht = data;

179 - n->network_viewer.count = 1;

180 - n->network_viewer.local_address_space = local_sockets_address_space(&n->local);

181 - n->network_viewer.remote_address_space = local_sockets_address_space(&n->remote);

232 + n->network_viewer.aggregated_key.pid = n->pid;

233 + n->network_viewer.aggregated_key.uid = n->uid;

234 + n->network_viewer.aggregated_key.direction = n->direction;

235 + n->network_viewer.aggregated_key.net_ns_inode = n->net_ns_inode;

236 + n->network_viewer.aggregated_key.state = n->state;

237

238 switch(n->direction) {

239 case SOCKET_DIRECTION_INBOUND:

240 case SOCKET_DIRECTION_LOCAL_INBOUND:

241 case SOCKET_DIRECTION_LISTEN:

187 - memset(&n->remote.ip, 0, sizeof(n->remote.ip));

188 - n->remote.port = 0;

242 + n->network_viewer.aggregated_key.server = n->local;

243 break;

244

245 case SOCKET_DIRECTION_OUTBOUND:

246 case SOCKET_DIRECTION_LOCAL_OUTBOUND:

193 - memset(&n->local.ip, 0, sizeof(n->local.ip));

194 - n->local.port = 0;

247 + n->network_viewer.aggregated_key.server = n->remote;

248 break;

249

250 case SOCKET_DIRECTION_NONE:

198 - return;

251 + break;

252 }

253

201 - n->inode = 0;

202 - n->local_ip_hash = 0;

203 - n->remote_ip_hash = 0;

204 - n->local_port_hash = 0;

205 - n->timer = 0;

206 - n->retransmits = 0;

207 - n->expires = 0;

208 - n->rqueue = 0;

209 - n->wqueue = 0;

210 - memset(&n->local_port_key, 0, sizeof(n->local_port_key));

211 -

212 - XXH64_hash_t hash = XXH3_64bits(n, sizeof(*n));

254 + n->network_viewer.aggregated_key.local_address_space = local_sockets_address_space(&n->local);

255 + n->network_viewer.aggregated_key.remote_address_space = local_sockets_address_space(&n->remote);

256 +}

257 +

258 +static void local_sockets_cb_to_json(LS_STATE *ls, LOCAL_SOCKET *n, void *data) {

259 + struct sockets_stats *st = data;

260 + populate_aggregated_key(n);

261 + local_socket_to_json_array(st, n, ls->proc_self_net_ns_inode, false);

262 +}

263 +

264 +#define KEEP_THE_BIGGER(a, b) (a) = ((a) < (b)) ? (b) : (a)

265 +#define KEEP_THE_SMALLER(a, b) (a) = ((a) > (b)) ? (b) : (a)

266 +#define SUM_THEM_ALL(a, b) (a) += (b)

267 +#define OR_THEM_ALL(a, b) (a) |= (b)

268 +

269 +static void local_sockets_cb_to_aggregation(LS_STATE *ls __maybe_unused, LOCAL_SOCKET *n, void *data) {

270 + SIMPLE_HASHTABLE_AGGREGATED_SOCKETS *ht = data;

271 +

272 + populate_aggregated_key(n);

273 + XXH64_hash_t hash = XXH3_64bits(&n->network_viewer.aggregated_key, sizeof(n->network_viewer.aggregated_key));

274 SIMPLE_HASHTABLE_SLOT_AGGREGATED_SOCKETS *sl = simple_hashtable_get_slot_AGGREGATED_SOCKETS(ht, hash, n, true);

275 LOCAL_SOCKET *t = SIMPLE_HASHTABLE_SLOT_DATA(sl);

276 if(t) {

277 t->network_viewer.count++;

278 +

279 + KEEP_THE_BIGGER(t->timer, n->timer);

280 + KEEP_THE_BIGGER(t->retransmits, n->retransmits);

281 + KEEP_THE_SMALLER(t->expires, n->expires);

282 + KEEP_THE_BIGGER(t->rqueue, n->rqueue);

283 + KEEP_THE_BIGGER(t->wqueue, n->wqueue);

284 +

285 + // The current number of consecutive retransmissions that have occurred for the most recently transmitted segment.

286 + SUM_THEM_ALL(t->info.tcp.tcpi_retransmits, n->info.tcp.tcpi_retransmits);

287 +

288 + // The total number of retransmissions that have occurred for the entire connection since it was established.

289 + SUM_THEM_ALL(t->info.tcp.tcpi_total_retrans, n->info.tcp.tcpi_total_retrans);

290 +

291 + // The total number of segments that have been retransmitted since the connection was established.

292 + SUM_THEM_ALL(t->info.tcp.tcpi_retrans, n->info.tcp.tcpi_retrans);

293 +

294 + // The number of keepalive probes sent

295 + SUM_THEM_ALL(t->info.tcp.tcpi_probes, n->info.tcp.tcpi_probes);

296 +

297 + // The number of times the retransmission timeout has been backed off.

298 + SUM_THEM_ALL(t->info.tcp.tcpi_backoff, n->info.tcp.tcpi_backoff);

299 +

300 + // A bitmask representing the TCP options currently enabled for the connection, such as SACK and Timestamps.

301 + OR_THEM_ALL(t->info.tcp.tcpi_options, n->info.tcp.tcpi_options);

302 +

303 + // The send window scale value used for this connection

304 + KEEP_THE_SMALLER(t->info.tcp.tcpi_snd_wscale, n->info.tcp.tcpi_snd_wscale);

305 +

306 + // The receive window scale value used for this connection

307 + KEEP_THE_SMALLER(t->info.tcp.tcpi_rcv_wscale, n->info.tcp.tcpi_rcv_wscale);

308 +

309 + // Retransmission timeout in milliseconds

310 + KEEP_THE_SMALLER(t->info.tcp.tcpi_rto, n->info.tcp.tcpi_rto);

311 +

312 + // The delayed acknowledgement timeout in milliseconds.

313 + KEEP_THE_SMALLER(t->info.tcp.tcpi_ato, n->info.tcp.tcpi_ato);

314 +

315 + // The maximum segment size for sending.

316 + KEEP_THE_SMALLER(t->info.tcp.tcpi_snd_mss, n->info.tcp.tcpi_snd_mss);

317 +

318 + // The maximum segment size for receiving.

319 + KEEP_THE_SMALLER(t->info.tcp.tcpi_rcv_mss, n->info.tcp.tcpi_rcv_mss);

320 +

321 + // The number of unacknowledged segments

322 + SUM_THEM_ALL(t->info.tcp.tcpi_unacked, n->info.tcp.tcpi_unacked);

323 +

324 + // The number of segments that have been selectively acknowledged

325 + SUM_THEM_ALL(t->info.tcp.tcpi_sacked, n->info.tcp.tcpi_sacked);

326 +

327 + // The number of segments that have been selectively acknowledged

328 + SUM_THEM_ALL(t->info.tcp.tcpi_sacked, n->info.tcp.tcpi_sacked);

329 +

330 + // The number of lost segments.

331 + SUM_THEM_ALL(t->info.tcp.tcpi_lost, n->info.tcp.tcpi_lost);

332 +

333 + // The number of forward acknowledgment segments.

334 + SUM_THEM_ALL(t->info.tcp.tcpi_fackets, n->info.tcp.tcpi_fackets);

335 +

336 + // The time in milliseconds since the last data was sent.

337 + KEEP_THE_SMALLER(t->info.tcp.tcpi_last_data_sent, n->info.tcp.tcpi_last_data_sent);

338 +

339 + // The time in milliseconds since the last acknowledgment was sent (not tracked in Linux, hence often zero).

340 + KEEP_THE_SMALLER(t->info.tcp.tcpi_last_ack_sent, n->info.tcp.tcpi_last_ack_sent);

341 +

342 + // The time in milliseconds since the last data was received.

343 + KEEP_THE_SMALLER(t->info.tcp.tcpi_last_data_recv, n->info.tcp.tcpi_last_data_recv);

344 +

345 + // The time in milliseconds since the last acknowledgment was received.

346 + KEEP_THE_SMALLER(t->info.tcp.tcpi_last_ack_recv, n->info.tcp.tcpi_last_ack_recv);

347 +

348 + // The path MTU for this connection

349 + KEEP_THE_SMALLER(t->info.tcp.tcpi_pmtu, n->info.tcp.tcpi_pmtu);

350 +

351 + // The slow start threshold for receiving

352 + KEEP_THE_SMALLER(t->info.tcp.tcpi_rcv_ssthresh, n->info.tcp.tcpi_rcv_ssthresh);

353 +

354 + // The slow start threshold for sending

355 + KEEP_THE_SMALLER(t->info.tcp.tcpi_snd_ssthresh, n->info.tcp.tcpi_snd_ssthresh);

356 +

357 + // The round trip time in milliseconds

358 + KEEP_THE_BIGGER(t->info.tcp.tcpi_rtt, n->info.tcp.tcpi_rtt);

359 +

360 + // The round trip time variance in milliseconds.

361 + KEEP_THE_BIGGER(t->info.tcp.tcpi_rttvar, n->info.tcp.tcpi_rttvar);

362 +

363 + // The size of the sending congestion window.

364 + KEEP_THE_SMALLER(t->info.tcp.tcpi_snd_cwnd, n->info.tcp.tcpi_snd_cwnd);

365 +

366 + // The maximum segment size that could be advertised.

367 + KEEP_THE_BIGGER(t->info.tcp.tcpi_advmss, n->info.tcp.tcpi_advmss);

368 +

369 + // The reordering metric

370 + KEEP_THE_SMALLER(t->info.tcp.tcpi_reordering, n->info.tcp.tcpi_reordering);

371 +

372 + // The receive round trip time in milliseconds.

373 + KEEP_THE_BIGGER(t->info.tcp.tcpi_rcv_rtt, n->info.tcp.tcpi_rcv_rtt);

374 +

375 + // The available space in the receive buffer.

376 + KEEP_THE_SMALLER(t->info.tcp.tcpi_rcv_space, n->info.tcp.tcpi_rcv_space);

377 }

378 else {

379 t = mallocz(sizeof(*t));

@@ -240,6 +400,10 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu

400 wb->content_type = CT_APPLICATION_JSON;

401 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);

402

403 + struct sockets_stats st = {

404 + .wb = wb,

405 + };

406 +

407 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);

408 buffer_json_member_add_string(wb, "type", "table");

409 buffer_json_member_add_time_t(wb, "update_every", 5);

@@ -328,9 +492,12 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu

492 .cmdline = true,

493 .comm = true,

494 .namespaces = true,

495 + .tcp_info = true,

496

497 .max_errors = 10,

498 + .max_concurrent_namespaces = 5,

499 },

500 + .spawn_server = spawn_srv,

501 .stats = { 0 },

502 .sockets_hashtable = { 0 },

503 .local_ips_hashtable = { 0 },

@@ -345,7 +512,7 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu

512 }

513 else {

514 ls.config.cb = local_sockets_cb_to_json;

348 - ls.config.data = wb;

515 + ls.config.data = &st;

516 }

517

518 local_sockets_process(&ls);

@@ -366,7 +533,7 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu

533 qsort(array, added, sizeof(LOCAL_SOCKET *), local_sockets_compar);

534

535 for(size_t i = 0; i < added ;i++) {

369 - local_socket_to_json_array(wb, array[i], proc_self_net_ns_inode, true);

536 + local_socket_to_json_array(&st, array[i], proc_self_net_ns_inode, true);

537 string_freez(array[i]->cmdline);

538 freez(array[i]);

539 }

@@ -555,14 +722,40 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu

722 // RRDF_FIELD_OPTS_NONE,

723 // NULL);

724

725 +

726 + // RTT

727 + buffer_rrdf_table_add_field(wb, field_id++, "RTT", aggregated ? "Max Smoothed Round Trip Time" : "Smoothed Round Trip Time",

728 + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,

729 + 2, "ms", st.max.tcpi_rtt / USEC_PER_MS, RRDF_FIELD_SORT_DESCENDING, NULL,

730 + RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,

731 + RRDF_FIELD_OPTS_VISIBLE,

732 + NULL);

733 +

734 + // Asymmetry RTT

735 + buffer_rrdf_table_add_field(wb, field_id++, "RecvRTT", aggregated ? "Max Receiver ACKs RTT" : "Receiver ACKs RTT",

736 + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,

737 + 2, "ms", st.max.tcpi_rcv_rtt / USEC_PER_MS, RRDF_FIELD_SORT_DESCENDING, NULL,

738 + RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,

739 + RRDF_FIELD_OPTS_VISIBLE,

740 + NULL);

741 +

742 + // Rentrasmissions

743 + buffer_rrdf_table_add_field(wb, field_id++, "Retrans", "Total Retransmissions",

744 + RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,

745 + 0, "packets", st.max.tcpi_total_retrans, RRDF_FIELD_SORT_DESCENDING, NULL,

746 + RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,

747 + RRDF_FIELD_OPTS_VISIBLE,

748 + NULL);

749 +

750 // Count

751 buffer_rrdf_table_add_field(wb, field_id++, "Count", "Number of sockets like this",

752 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,

561 - 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,

562 - RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,

753 + 0, "sockets", NAN, RRDF_FIELD_SORT_DESCENDING, NULL,

754 + RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_NONE,

755 aggregated ? (RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_STICKY) : RRDF_FIELD_OPTS_NONE,

756 NULL);

757 }

758 +

759 buffer_json_object_close(wb); // columns

760 buffer_json_member_add_string(wb, "default_sort_column", aggregated ? "Count" : "Direction");

761

@@ -747,18 +940,28 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {

940

941 uc = system_usernames_cache_init();

942

943 + spawn_srv = spawn_server_create("setns", local_sockets_spawn_server_callback, argc, (const char **)argv);

944 + if(spawn_srv == NULL) {

945 + fprintf(stderr, "Cannot create spawn server.\n");

946 + exit(1);

947 + }

948 +

949 // ----------------------------------------------------------------------------------------------------------------

950

951 if(argc == 2 && strcmp(argv[1], "debug") == 0) {

753 - bool cancelled = false;

754 - usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;

755 - char buf[] = "network-connections sockets:aggregated";

756 - network_viewer_function("123", buf, &stop_monotonic_ut, &cancelled,

757 - NULL, HTTP_ACCESS_ALL, NULL, NULL);

758 -

759 - char buf2[] = "network-connections sockets:detailed";

760 - network_viewer_function("123", buf2, &stop_monotonic_ut, &cancelled,

761 - NULL, HTTP_ACCESS_ALL, NULL, NULL);

952 +// for(int i = 0; i < 100; i++) {

953 + bool cancelled = false;

954 + usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;

955 + char buf[] = "network-connections sockets:aggregated";

956 + network_viewer_function("123", buf, &stop_monotonic_ut, &cancelled,

957 + NULL, HTTP_ACCESS_ALL, NULL, NULL);

958 +

959 + char buf2[] = "network-connections sockets:detailed";

960 + network_viewer_function("123", buf2, &stop_monotonic_ut, &cancelled,

961 + NULL, HTTP_ACCESS_ALL, NULL, NULL);

962 +// }

963 +

964 + spawn_server_destroy(spawn_srv);

965 exit(1);

966 }

967

@@ -799,5 +1002,8 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {

1002 }

1003 }

1004

1005 + spawn_server_destroy(spawn_srv);

1006 + spawn_srv = NULL;

1007 +

1008 return 0;

1009 }

src/collectors/nfacct.plugin/plugin_nfacct.c

+1 -1

@@ -809,7 +809,7 @@ int main(int argc, char **argv) {

809

810 nfacct_signals();

811

812 - errno = 0;

812 + errno_clear();

813

814 if(freq >= netdata_update_every)

815 netdata_update_every = freq;

src/collectors/perf.plugin/perf_plugin.c

+1 -1

@@ -1288,7 +1288,7 @@ int main(int argc, char **argv) {

1288

1289 parse_command_line(argc, argv);

1290

1291 - errno = 0;

1291 + errno_clear();

1292

1293 if(freq >= update_every)

1294 update_every = freq;

src/collectors/plugins.d/local_listeners.c

+25 -1

@@ -15,6 +15,14 @@ static const char *protocol_name(LOCAL_SOCKET *n) {

15 else

16 return "UNKNOWN_IPV4";

17 }

18 + else if(is_local_socket_ipv46(n)) {

19 + if (n->local.protocol == IPPROTO_TCP)

20 + return "TCP46";

21 + else if(n->local.protocol == IPPROTO_UDP)

22 + return "UDP46";

23 + else

24 + return "UNKNOWN_IPV46";

25 + }

26 else if(n->local.family == AF_INET6) {

27 if (n->local.protocol == IPPROTO_TCP)

28 return "TCP6";

@@ -35,6 +43,10 @@ static void print_local_listeners(LS_STATE *ls __maybe_unused, LOCAL_SOCKET *n,

43 ipv4_address_to_txt(n->local.ip.ipv4, local_address);

44 ipv4_address_to_txt(n->remote.ip.ipv4, remote_address);

45 }

46 + else if(is_local_socket_ipv46(n)) {

47 + strncpyz(local_address, "*", sizeof(local_address) - 1);

48 + remote_address[0] = '\0';

49 + }

50 else if(n->local.family == AF_INET6) {

51 ipv6_address_to_txt(&n->local.ip.ipv6, local_address);

52 ipv6_address_to_txt(&n->remote.ip.ipv6, remote_address);

@@ -93,8 +105,10 @@ int main(int argc, char **argv) {

105 .cmdline = true,

106 .comm = false,

107 .namespaces = true,

108 + .tcp_info = false,

109

110 .max_errors = 10,

111 + .max_concurrent_namespaces = 10,

112

113 .cb = print_local_listeners,

114 .data = NULL,

@@ -212,6 +226,7 @@ int main(int argc, char **argv) {

226 ls.config.comm = true;

227 ls.config.cmdline = true;

228 ls.config.namespaces = true;

229 + ls.config.tcp_info = true;

230 ls.config.uid = true;

231 ls.config.max_errors = SIZE_MAX;

232 ls.config.cb = print_local_listeners_debug;

@@ -276,8 +291,17 @@ int main(int argc, char **argv) {

291 }

292 }

293

294 + SPAWN_SERVER *spawn_server = spawn_server_create(NULL, local_sockets_spawn_server_callback, argc, (const char **)argv);

295 + if(spawn_server == NULL) {

296 + fprintf(stderr, "Cannot create spawn server.\n");

297 + exit(1);

298 + }

299 + ls.spawn_server = spawn_server;

300 +

301 local_sockets_process(&ls);

302

303 + spawn_server_destroy(spawn_server);

304 +

305 getrusage(RUSAGE_SELF, &ended);

306

307 if(debug) {

@@ -285,7 +309,7 @@ int main(int argc, char **argv) {

309 unsigned long long system = ended.ru_stime.tv_sec * 1000000ULL + ended.ru_stime.tv_usec - started.ru_stime.tv_sec * 1000000ULL + started.ru_stime.tv_usec;

310 unsigned long long total = user + system;

311

288 - fprintf(stderr, "CPU Usage %llu user, %llu system, %llu total\n", user, system, total);

312 + fprintf(stderr, "CPU Usage %llu user, %llu system, %llu total, %zu namespaces, %zu nl requests (without namespaces)\n", user, system, total, ls.stats.namespaces_found, ls.stats.mnl_sends);

313 }

314

315 return 0;

src/collectors/plugins.d/plugins_d.c

+12 -22

@@ -68,23 +68,15 @@ static void pluginsd_worker_thread_cleanup(void *pptr) {

68 cd->unsafe.running = false;

69 cd->unsafe.thread = 0;

71 - pid_t pid = cd->unsafe.pid;

71 cd->unsafe.pid = 0;

74 - spinlock_unlock(&cd->unsafe.spinlock);

75 -

76 - if (pid) {

77 - siginfo_t info;

78 - netdata_log_info("PLUGINSD: 'host:%s', killing data collection child process with pid %d",

79 - rrdhost_hostname(cd->host), pid);

73 + POPEN_INSTANCE *pi = cd->unsafe.pi;

74 + cd->unsafe.pi = NULL;

81 - if (killpid(pid) != -1) {

82 - netdata_log_info("PLUGINSD: 'host:%s', waiting for data collection child process pid %d to exit...",

83 - rrdhost_hostname(cd->host), pid);

76 + spinlock_unlock(&cd->unsafe.spinlock);

85 - netdata_waitid(P_PID, (id_t)pid, &info, WEXITED);

86 - }

87 - }

78 + if (pi)

79 + spawn_popen_kill(pi);

80 }

82 #define SERIAL_FAILURES_THRESHOLD 10

@@ -160,14 +152,13 @@ static void *pluginsd_worker_thread(void *arg) {

152 size_t count = 0;

153

154 while(service_running(SERVICE_COLLECTORS)) {

163 - FILE *fp_child_input = NULL;

164 - FILE *fp_child_output = netdata_popen(cd->cmd, &cd->unsafe.pid, &fp_child_input);

165 -

166 - if(unlikely(!fp_child_input || !fp_child_output)) {

155 + cd->unsafe.pi = spawn_popen_run(cd->cmd);

156 + if(!cd->unsafe.pi) {

157 netdata_log_error("PLUGINSD: 'host:%s', cannot popen(\"%s\", \"r\").",

158 rrdhost_hostname(cd->host), cd->cmd);

159 break;

160 }

161 + cd->unsafe.pid = spawn_server_instance_pid(cd->unsafe.pi->si);

162

163 nd_log(NDLS_DAEMON, NDLP_DEBUG,

164 "PLUGINSD: 'host:%s' connected to '%s' running on pid %d",

@@ -190,15 +181,14 @@ static void *pluginsd_worker_thread(void *arg) {

181 };

182 ND_LOG_STACK_PUSH(lgs);

183

193 - count = pluginsd_process(cd->host, cd, fp_child_input, fp_child_output, 0);

184 + count = pluginsd_process(cd->host, cd, cd->unsafe.pi->child_stdin_fp, cd->unsafe.pi->child_stdout_fp, 0);

185

186 nd_log(NDLS_DAEMON, NDLP_DEBUG,

187 "PLUGINSD: 'host:%s', '%s' (pid %d) disconnected after %zu successful data collections (ENDs).",

188 rrdhost_hostname(cd->host), cd->fullfilename, cd->unsafe.pid, count);

189

199 - killpid(cd->unsafe.pid);

200 -

201 - int worker_ret_code = netdata_pclose(fp_child_input, fp_child_output, cd->unsafe.pid);

190 + int worker_ret_code = spawn_popen_kill(cd->unsafe.pi);

191 + cd->unsafe.pi = NULL;

192

193 if(likely(worker_ret_code == 0))

194 pluginsd_worker_thread_handle_success(cd);

@@ -273,7 +263,7 @@ void *pluginsd_main(void *ptr) {

263 if (unlikely(!service_running(SERVICE_COLLECTORS)))

264 break;

265

276 - errno = 0;

266 + errno_clear();

267 DIR *dir = opendir(directory_name);

268 if (unlikely(!dir)) {

269 if (directory_errors[idx] != errno) {

src/collectors/plugins.d/plugins_d.h

@@ -34,6 +34,7 @@ struct plugind {

34 bool running; // do not touch this structure after setting this to 1

35 bool enabled; // if this is enabled or not

36 ND_THREAD *thread;

37 + POPEN_INSTANCE *pi;

38 pid_t pid;

39 } unsafe;

src/collectors/plugins.d/pluginsd_internals.c

+1 -1

@@ -13,7 +13,7 @@ ssize_t send_to_plugin(const char *txt, void *data) {

13 return h2o_stream_write(parser->h2o_ctx, txt, strlen(txt));

14 #endif

16 - errno = 0;

16 + errno_clear();

17 spinlock_lock(&parser->writer.spinlock);

18 ssize_t bytes = -1;

src/collectors/proc.plugin/proc_meminfo.c

+2 -2

@@ -29,7 +29,7 @@ int do_proc_meminfo(int update_every, usec_t dt) {

30 static ARL_BASE *arl_base = NULL;

31 static ARL_ENTRY *arl_hwcorrupted = NULL, *arl_memavailable = NULL, *arl_hugepages_total = NULL,

32 - *arl_zswapped = NULL, *arl_high_low = NULL, *arl_cma_total = NULL,

32 + *arl_zswapped = NULL, *arl_high_low = NULL,

33 *arl_directmap4k = NULL, *arl_directmap2m = NULL, *arl_directmap4m = NULL, *arl_directmap1g = NULL;

35 static unsigned long long

@@ -189,7 +189,7 @@ int do_proc_meminfo(int update_every, usec_t dt) {

189 arl_expect(arl_base, "FilePmdMapped", &FilePmdMapped);

190

191 // CONFIG_CMA

192 - arl_cma_total = arl_expect(arl_base, "CmaTotal", &CmaTotal);

192 + arl_expect(arl_base, "CmaTotal", &CmaTotal);

193 arl_expect(arl_base, "CmaFree", &CmaFree);

194

195 // CONFIG_UNACCEPTED_MEMORY

src/collectors/systemd-journal.plugin/systemd-journal.c

+1 -1

@@ -1037,7 +1037,7 @@ static ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_one_file(

1037 struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) {

1038

1039 sd_journal *j = NULL;

1040 - errno = 0;

1040 + errno_clear();

1041

1042 fstat_cache_enable_on_thread();

1043

src/collectors/tc.plugin/plugin_tc.c

+10 -17

@@ -834,7 +834,7 @@ static inline void tc_split_words(char *str, char **words, int max_words) {

834 while(i < max_words) words[i++] = NULL;

835 }

836

837 -static pid_t tc_child_pid = 0;

837 +static POPEN_INSTANCE *tc_child_instance = NULL;

838

839 static void tc_main_cleanup(void *pptr) {

840 struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr);

@@ -847,16 +847,10 @@ static void tc_main_cleanup(void *pptr) {

847

848 collector_info("cleaning up...");

849

850 - if(tc_child_pid) {

851 - collector_info("TC: killing with SIGTERM tc-qos-helper process %d", tc_child_pid);

852 - if(killpid(tc_child_pid) != -1) {

853 - siginfo_t info;

854 -

855 - collector_info("TC: waiting for tc plugin child process pid %d to exit...", tc_child_pid);

856 - netdata_waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED);

857 - }

858 -

859 - tc_child_pid = 0;

850 + if(tc_child_instance) {

851 + collector_info("TC: stopping the running tc-qos-helper script");

852 + int code = spawn_popen_wait(tc_child_instance); (void)code;

853 + tc_child_instance = NULL;

854 }

855

856 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

@@ -921,21 +915,20 @@ void *tc_main(void *ptr) {

915 char *tc_script = config_get("plugin:tc", "script to run to get tc values", command);

916

917 while(service_running(SERVICE_COLLECTORS)) {

924 - FILE *fp_child_input, *fp_child_output;

918 struct tc_device *device = NULL;

919 struct tc_class *class = NULL;

920

921 snprintfz(command, TC_LINE_MAX, "exec %s %d", tc_script, localhost->rrd_update_every);

922 netdata_log_debug(D_TC_LOOP, "executing '%s'", command);

923

931 - fp_child_output = netdata_popen(command, (pid_t *)&tc_child_pid, &fp_child_input);

932 - if(unlikely(!fp_child_output)) {

924 + tc_child_instance = spawn_popen_run(command);

925 + if(!tc_child_instance) {

926 collector_error("TC: Cannot popen(\"%s\", \"r\").", command);

927 goto cleanup;

928 }

929

930 char buffer[TC_LINE_MAX+1] = "";

938 - while(fgets(buffer, TC_LINE_MAX, fp_child_output) != NULL) {

931 + while(fgets(buffer, TC_LINE_MAX, tc_child_instance->child_stdout_fp) != NULL) {

932 if(unlikely(!service_running(SERVICE_COLLECTORS))) break;

933

934 buffer[TC_LINE_MAX] = '\0';

@@ -1142,8 +1135,8 @@ void *tc_main(void *ptr) {

1135 }

1136

1137 // fgets() failed or loop broke

1145 - int code = netdata_pclose(fp_child_input, fp_child_output, (pid_t)tc_child_pid);

1146 - tc_child_pid = 0;

1138 + int code = spawn_popen_kill(tc_child_instance);

1139 + tc_child_instance = NULL;

1140

1141 if(unlikely(device)) {

1142 // tc_device_free(device);

src/collectors/xenstat.plugin/xenstat_plugin.c

+1 -1

@@ -986,7 +986,7 @@ int main(int argc, char **argv) {

986 netdata_log_error("xenstat.plugin: ignoring parameter '%s'", argv[i]);

987 }

988

989 - errno = 0;

989 + errno_clear();

990

991 if(freq >= netdata_update_every)

992 netdata_update_every = freq;

src/daemon/analytics.c

+8 -14

@@ -326,18 +326,15 @@ void analytics_alarms_notifications(void)

326

327 strcat(script, " dump_methods");

328

329 - pid_t command_pid;

330 -

329 netdata_log_debug(D_ANALYTICS, "Executing %s", script);

330

331 BUFFER *b = buffer_create(1000, NULL);

332 int cnt = 0;

335 - FILE *fp_child_input;

336 - FILE *fp_child_output = netdata_popen(script, &command_pid, &fp_child_input);

337 - if (fp_child_output) {

333 + POPEN_INSTANCE *instance = spawn_popen_run(script);

334 + if (instance) {

335 char line[200 + 1];

336

340 - while (fgets(line, 200, fp_child_output) != NULL) {

337 + while (fgets(line, 200, instance->child_stdout_fp) != NULL) {

338 char *end = line;

339 while (*end && *end != '\n')

340 end++;

@@ -350,7 +347,7 @@ void analytics_alarms_notifications(void)

347

348 cnt++;

349 }

353 - netdata_pclose(fp_child_input, fp_child_output, command_pid);

350 + spawn_popen_wait(instance);

351 }

352 freez(script);

353

@@ -1001,8 +998,6 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) {

998 char *command_to_run = mallocz(

999 sizeof(char) * (strlen(statistic->action) + strlen(action_result) + strlen(action_data) + strlen(as_script) +

1000 analytics_data.data_length + (ANALYTICS_NO_OF_ITEMS * 3) + 15));

1004 - pid_t command_pid;

1005 -

1001 sprintf(

1002 command_to_run,

1003 "%s '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' ",

@@ -1055,12 +1050,11 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) {

1050 "%s '%s' '%s' '%s'",

1051 as_script, statistic->action, action_result, action_data);

1052

1058 - FILE *fp_child_input;

1059 - FILE *fp_child_output = netdata_popen(command_to_run, &command_pid, &fp_child_input);

1060 - if (fp_child_output) {

1053 + POPEN_INSTANCE *instance = spawn_popen_run(command_to_run);

1054 + if (instance) {

1055 char buffer[4 + 1];

1062 - char *s = fgets(buffer, 4, fp_child_output);

1063 - int exit_code = netdata_pclose(fp_child_input, fp_child_output, command_pid);

1056 + char *s = fgets(buffer, 4, instance->child_stdout_fp);

1057 + int exit_code = spawn_popen_wait(instance);

1058 if (exit_code)

1059

1060 nd_log(NDLS_DAEMON, NDLP_NOTICE,

src/daemon/buildinfo.c

+21

@@ -75,6 +75,7 @@ typedef enum __attribute__((packed)) {

75 BIB_LIB_LIBCAP,

76 BIB_LIB_LIBCRYPTO,

77 BIB_LIB_LIBYAML,

78 + BIB_LIB_LIBMNL,

79 BIB_PLUGIN_APPS,

80 BIB_PLUGIN_LINUX_CGROUPS,

81 BIB_PLUGIN_LINUX_CGROUP_NETWORK,

@@ -698,6 +699,14 @@ static struct {

699 .json = "libyaml",

700 .value = NULL,

701 },

702 + [BIB_LIB_LIBMNL] = {

703 + .category = BIC_LIBS,

704 + .type = BIT_BOOLEAN,

705 + .analytics = "libmnl",

706 + .print = "libmnl (library for working with netfilter)",

707 + .json = "libmnl",

708 + .value = NULL,

709 + },

710 [BIB_PLUGIN_APPS] = {

711 .category = BIC_PLUGINS,

712 .type = BIT_BOOLEAN,

@@ -1177,6 +1186,9 @@ __attribute__((constructor)) void initialize_build_info(void) {

1186 #ifdef HAVE_LIBYAML

1187 build_info_set_status(BIB_LIB_LIBYAML, true);

1188 #endif

1189 +#ifdef HAVE_LIBMNL

1190 + build_info_set_status(BIB_LIB_LIBMNL, true);

1191 +#endif

1192

1193 #ifdef ENABLE_PLUGIN_APPS

1194 build_info_set_status(BIB_PLUGIN_APPS, true);

@@ -1278,9 +1290,18 @@ static void populate_system_info(void) {

1290 system_info = localhost->system_info;

1291 }

1292 else {

1293 + bool started_spawn_server = false;

1294 + if(!netdata_main_spawn_server) {

1295 + started_spawn_server = true;

1296 + netdata_main_spawn_server_init(NULL, 0, NULL);

1297 + }

1298 +

1299 system_info = callocz(1, sizeof(struct rrdhost_system_info));

1300 get_system_info(system_info);

1301 free_system_info = true;

1302 +

1303 + if(started_spawn_server)

1304 + netdata_main_spawn_server_cleanup();

1305 }

1306

1307 build_info_set_value_strdupz(BIB_OS_KERNEL_NAME, system_info->kernel_name);

src/daemon/common.c

+1 -1

@@ -44,7 +44,7 @@ long get_netdata_cpus(void) {

45 long cores_user_configured = config_get_number(CONFIG_SECTION_GLOBAL, "cpu cores", processors);

47 - errno = 0;

47 + errno_clear();

48 internal_error(true,

49 "System CPUs: %ld, ("

50 "system: %ld, cgroups cpuset v1: %ld, cgroups cpuset v2: %ld, netdata.conf: %ld"

src/daemon/common.h

-3

@@ -84,9 +84,6 @@

85 // global GUID map functions

87 -// netdata agent spawn server

88 -#include "spawn/spawn.h"

89 -

87 // the netdata daemon

88 #include "daemon.h"

89 #include "main.h"

src/daemon/daemon.c

+3 -3

@@ -381,14 +381,14 @@ static void sched_setscheduler_set(void) {

381 priority = (int)config_get_number(CONFIG_SECTION_GLOBAL, "process scheduling priority", priority);

382

383 #ifdef HAVE_SCHED_GET_PRIORITY_MIN

384 - errno = 0;

384 + errno_clear();

385 if(priority < sched_get_priority_min(policy)) {

386 netdata_log_error("scheduler %s (%d) priority %d is below the minimum %d. Using the minimum.", name, policy, priority, sched_get_priority_min(policy));

387 priority = sched_get_priority_min(policy);

388 }

389 #endif

390 #ifdef HAVE_SCHED_GET_PRIORITY_MAX

391 - errno = 0;

391 + errno_clear();

392 if(priority > sched_get_priority_max(policy)) {

393 netdata_log_error("scheduler %s (%d) priority %d is above the maximum %d. Using the maximum.", name, policy, priority, sched_get_priority_max(policy));

394 priority = sched_get_priority_max(policy);

@@ -407,7 +407,7 @@ static void sched_setscheduler_set(void) {

407 .sched_priority = priority

408 };

409

410 - errno = 0;

410 + errno_clear();

411 i = sched_setscheduler(0, policy, &param);

412 if(i != 0) {

413 netdata_log_error("Cannot adjust netdata scheduling policy to %s (%d), with priority %d. Falling back to nice.",

src/daemon/main.c

+15 -64

@@ -26,7 +26,6 @@ int libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;

26 bool ieee754_doubles = false;

27 time_t netdata_start_time = 0;

28 struct netdata_static_thread *static_threads;

29 -bool i_am_the_spawn_server = false;

30 struct config netdata_config = {

31 .first_section = NULL,

@@ -325,9 +324,6 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

324 void web_client_cache_destroy(void);

325

326 void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) {

328 - if (i_am_the_spawn_server)

329 - exit(ret);

330 -

327 watcher_shutdown_begin();

328

329 nd_log_limits_unlimited();

@@ -490,9 +486,12 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re

486 #endif

487 watcher_step_complete(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);

488

489 + netdata_main_spawn_server_cleanup();

490 + watcher_step_complete(WATCHER_STEP_ID_DESTROY_MAIN_SPAWN_SERVER);

491 +

492 (void) unlink(agent_incomplete_shutdown_file);

493 watcher_step_complete(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE);

495 -

494 +

495 watcher_shutdown_end();

496 watcher_thread_stop();

497

@@ -621,39 +620,6 @@ void web_server_config_options(void)

620 }

621 }

622

624 -

625 -// killpid kills pid with SIGTERM.

626 -int killpid(pid_t pid) {

627 - int ret;

628 - netdata_log_debug(D_EXIT, "Request to kill pid %d", pid);

629 -

630 - int signal = SIGTERM;

631 -//#ifdef NETDATA_INTERNAL_CHECKS

632 -// if(service_running(SERVICE_COLLECTORS))

633 -// signal = SIGABRT;

634 -//#endif

635 -

636 - errno = 0;

637 - ret = kill(pid, signal);

638 - if (ret == -1) {

639 - switch(errno) {

640 - case ESRCH:

641 - // We wanted the process to exit so just let the caller handle.

642 - return ret;

643 -

644 - case EPERM:

645 - netdata_log_error("Cannot kill pid %d, but I do not have enough permissions.", pid);

646 - break;

647 -

648 - default:

649 - netdata_log_error("Cannot kill pid %d, but I received an error.", pid);

650 - break;

651 - }

652 - }

653 -

654 - return ret;

655 -}

656 -

623 static void set_nofile_limit(struct rlimit *rl) {

624 // get the num files allowed

625 if(getrlimit(RLIMIT_NOFILE, rl) != 0) {

@@ -1333,7 +1299,7 @@ static void post_conf_load(char **user)

1299 }

1300

1301 static bool load_netdata_conf(char *filename, char overwrite_used, char **user) {

1336 - errno = 0;

1302 + errno_clear();

1303

1304 int ret = 0;

1305

@@ -1380,15 +1346,12 @@ int get_system_info(struct rrdhost_system_info *system_info) {

1346 return 1;

1347 }

1348

1383 - pid_t command_pid;

1384 -

1385 - FILE *fp_child_input;

1386 - FILE *fp_child_output = netdata_popen(script, &command_pid, &fp_child_input);

1387 - if(fp_child_output) {

1349 + POPEN_INSTANCE *instance = spawn_popen_run(script);

1350 + if(instance) {

1351 char line[200 + 1];

1352 // Removed the double strlens, if the Coverity tainted string warning reappears I'll revert.

1353 // One time init code, but I'm curious about the warning...

1391 - while (fgets(line, 200, fp_child_output) != NULL) {

1354 + while (fgets(line, 200, instance->child_stdout_fp) != NULL) {

1355 char *value=line;

1356 while (*value && *value != '=') value++;

1357 if (*value=='=') {

@@ -1407,7 +1370,7 @@ int get_system_info(struct rrdhost_system_info *system_info) {

1370 }

1371 }

1372 }

1410 - netdata_pclose(fp_child_input, fp_child_output, command_pid);

1373 + spawn_popen_wait(instance);

1374 }

1375 freez(script);

1376 #else

@@ -1464,15 +1427,12 @@ int unittest_prepare_rrd(char **user) {

1427 return 0;

1428 }

1429

1467 -int netdata_main(int argc, char **argv)

1468 -{

1469 - analytics_init();

1430 +int netdata_main(int argc, char **argv) {

1431 + clocks_init();

1432 string_init();

1433 + analytics_init();

1434

1472 - // initialize the system clocks

1473 - clocks_init();

1435 netdata_start_time = now_realtime_sec();

1475 -

1436 usec_t started_ut = now_monotonic_usec();

1437 usec_t last_ut = started_ut;

1438 const char *prev_msg = NULL;

@@ -1495,13 +1455,6 @@ int netdata_main(int argc, char **argv)

1455 // set the name for logging

1456 program_name = "netdata";

1457

1498 - if (argc > 1 && strcmp(argv[1], SPAWN_SERVER_COMMAND_LINE_ARGUMENT) == 0) {

1499 - // don't run netdata, this is the spawn server

1500 - i_am_the_spawn_server = true;

1501 - spawn_server();

1502 - exit(0);

1503 - }

1504 -

1458 // parse options

1459 {

1460 int num_opts = sizeof(option_definitions) / sizeof(struct option_def);

@@ -1966,7 +1919,7 @@ int netdata_main(int argc, char **argv)

1919 if (close_open_fds == true) {

1920 // close all open file descriptors, except the standard ones

1921 // the caller may have left open files (lxc-attach has this issue)

1969 - for_each_open_fd(OPEN_FD_ACTION_CLOSE, OPEN_FD_EXCLUDE_STDIN | OPEN_FD_EXCLUDE_STDOUT | OPEN_FD_EXCLUDE_STDERR);

1922 + os_close_all_non_std_open_fds_except(NULL, 0);

1923 }

1924

1925 if(!config_loaded) {

@@ -2196,6 +2149,7 @@ int netdata_main(int argc, char **argv)

2149 (void)dont_fork;

2150 #endif

2151

2152 + netdata_main_spawn_server_init("plugins", argc, (const char **)argv);

2153 watcher_thread_start();

2154

2155 // init sentry

@@ -2228,10 +2182,6 @@ int netdata_main(int argc, char **argv)

2182 // fork the spawn server

2183 delta_startup_time("fork the spawn server");

2184

2231 -#ifndef OS_WINDOWS

2232 - spawn_init();

2233 -#endif

2234 -

2185 /*

2186 * Libuv uv_spawn() uses SIGCHLD internally:

2187 * https://github.com/libuv/libuv/blob/cc51217a317e96510fbb284721d5e6bc2af31e33/src/unix/process.c#L485

@@ -2288,6 +2238,7 @@ int netdata_main(int argc, char **argv)

2238

2239 if (claiming_pending_arguments)

2240 claim_agent(claiming_pending_arguments, false, NULL);

2241 +

2242 load_claiming_state();

2243

2244 // ------------------------------------------------------------------------

src/daemon/main.h

-1

@@ -8,7 +8,6 @@

8 extern struct config netdata_config;

10 void cancel_main_threads(void);

11 -int killpid(pid_t pid);

12 typedef enum {

13 ABILITY_DATA_QUERIES = (1 << 0),

src/daemon/signals.c

+38 -53

@@ -118,61 +118,45 @@ void signals_reset(void) {

118 }

119 }

120

121 -// reap_child reaps the child identified by pid.

122 -static void reap_child(pid_t pid) {

123 - siginfo_t i;

124 -

125 - errno = 0;

126 - netdata_log_debug(D_CHILDS, "SIGNAL: reap_child(%d)...", pid);

127 - if (netdata_waitid(P_PID, (id_t)pid, &i, WEXITED|WNOHANG) == -1) {

128 - if (errno != ECHILD)

129 - netdata_log_error("SIGNAL: waitid(%d): failed to wait for child", pid);

130 - else

131 - netdata_log_info("SIGNAL: waitid(%d): failed - it seems the child is already reaped", pid);

132 - return;

133 - }

134 - else if (i.si_pid == 0) {

135 - // Process didn't exit, this shouldn't happen.

136 - netdata_log_error("SIGNAL: waitid(%d): reports pid 0 - child has not exited", pid);

137 - return;

138 - }

121 +static void sigchild_handle() {

122 + int status;

123 + pid_t pid;

124

140 - switch (i.si_code) {

141 - case CLD_EXITED:

142 - netdata_log_info("SIGNAL: reap_child(%d) exited with code: %d", pid, i.si_status);

143 - break;

144 - case CLD_KILLED:

145 - netdata_log_info("SIGNAL: reap_child(%d) killed by signal: %d", pid, i.si_status);

146 - break;

147 - case CLD_DUMPED:

148 - netdata_log_info("SIGNAL: reap_child(%d) dumped core by signal: %d", pid, i.si_status);

149 - break;

150 - case CLD_STOPPED:

151 - netdata_log_info("SIGNAL: reap_child(%d) stopped by signal: %d", pid, i.si_status);

125 + // Loop to check for exited child processes

126 + while ((pid = waitpid((pid_t)(-1), &status, WNOHANG)) != 0) {

127 + if(pid == -1)

128 break;

153 - case CLD_TRAPPED:

154 - netdata_log_info("SIGNAL: reap_child(%d) trapped by signal: %d", pid, i.si_status);

155 - break;

156 - case CLD_CONTINUED:

157 - netdata_log_info("SIGNAL: reap_child(%d) continued by signal: %d", pid, i.si_status);

158 - break;

159 - default:

160 - netdata_log_info("SIGNAL: reap_child(%d) gave us a SIGCHLD with code %d and status %d.", pid, i.si_code, i.si_status);

161 - break;

162 - }

163 -}

164 -

165 -// reap_children reaps all pending children which are not managed by myp.

166 -static void reap_children() {

167 - siginfo_t i;

168 -

169 - while(1) {

170 - i.si_pid = 0;

171 - if (netdata_waitid(P_ALL, (id_t)0, &i, WEXITED|WNOHANG|WNOWAIT) == -1 || i.si_pid == 0)

172 - // nothing to do

173 - return;

129

175 - reap_child(i.si_pid);

130 + if(WIFEXITED(status)) {

131 + nd_log(NDLS_DAEMON, NDLP_INFO,

132 + "DAEMON: child with pid %d exited normally with exit code %d",

133 + pid, WEXITSTATUS(status));

134 + }

135 + else if(WIFSIGNALED(status)) {

136 + if(WCOREDUMP(status))

137 + nd_log(NDLS_DAEMON, NDLP_INFO,

138 + "DAEMON: child with pid %d coredump'd due to signal %d",

139 + pid, WTERMSIG(status));

140 + else

141 + nd_log(NDLS_DAEMON, NDLP_INFO,

142 + "DAEMON: child with pid %d killed by signal %d",

143 + pid, WTERMSIG(status));

144 + }

145 + else if(WIFSTOPPED(status)) {

146 + nd_log(NDLS_DAEMON, NDLP_INFO,

147 + "DAEMON: child with pid %d stopped due to signal %d",

148 + pid, WSTOPSIG(status));

149 + }

150 + else if(WIFCONTINUED(status)) {

151 + nd_log(NDLS_DAEMON, NDLP_INFO,

152 + "DAEMON: child with pid %d continued due to signal %d",

153 + pid, SIGCONT);

154 + }

155 + else {

156 + nd_log(NDLS_COLLECTORS, NDLP_INFO,

157 + "DAEMON: child with pid %d reports unhandled status",

158 + pid);

159 + }

160 }

161 }

162

@@ -183,6 +167,7 @@ void signals_handle(void) {

167 // is delivered that either terminates the process or causes the invocation

168 // of a signal-catching function.

169 if(pause() == -1 && errno == EINTR) {

170 + errno_clear();

171

172 // loop once, but keep looping while signals are coming in

173 // this is needed because a few operations may take some time

@@ -227,7 +212,7 @@ void signals_handle(void) {

212 break;

213

214 case NETDATA_SIGNAL_CHILD:

230 - reap_children();

215 + sigchild_handle();

216 break;

217

218 default:

src/daemon/static_threads.c

-11

@@ -30,11 +30,7 @@ const struct netdata_static_thread static_threads_common[] = {

30 .name = "HEALTH",

31 .config_section = NULL,

32 .config_name = NULL,

33 -#ifdef OS_WINDOWS

34 - .enabled = 0,

35 -#else

33 .enabled = 1,

37 -#endif

34 .thread = NULL,

35 .init_routine = NULL,

36 .start_routine = health_main

@@ -74,11 +70,7 @@ const struct netdata_static_thread static_threads_common[] = {

70 .name = "PLUGINSD",

71 .config_section = NULL,

72 .config_name = NULL,

77 -#ifdef OS_WINDOWS

78 - .enabled = 0,

79 -#else

73 .enabled = 1,

81 -#endif

74 .thread = NULL,

75 .init_routine = NULL,

76 .start_routine = pluginsd_main

@@ -101,8 +93,6 @@ const struct netdata_static_thread static_threads_common[] = {

93 .init_routine = NULL,

94 .start_routine = statsd_main

95 },

104 -#ifndef OS_WINDOWS

105 - // this crashes the debugger under windows

96 {

97 .name = "EXPORTING",

98 .config_section = NULL,

@@ -112,7 +102,6 @@ const struct netdata_static_thread static_threads_common[] = {

102 .init_routine = NULL,

103 .start_routine = exporting_main

104 },

115 -#endif

105 {

106 .name = "SNDR[localhost]",

107 .config_section = NULL,

src/daemon/watcher.c

@@ -151,6 +151,8 @@ void watcher_thread_start() {

151 "remove pid file";

152 watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg =

153 "free openssl structures";

154 + watcher_steps[WATCHER_STEP_ID_DESTROY_MAIN_SPAWN_SERVER].msg =

155 + "destroy main spawn server";

156 watcher_steps[WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE].msg =

157 "remove incomplete shutdown file";

158

src/daemon/watcher.h

@@ -30,6 +30,7 @@ typedef enum {

30 WATCHER_STEP_ID_CLOSE_SQL_DATABASES,

31 WATCHER_STEP_ID_REMOVE_PID_FILE,

32 WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES,

33 + WATCHER_STEP_ID_DESTROY_MAIN_SPAWN_SERVER,

34 WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE,

36 // Always keep this as the last enum value

src/daemon/win_system-info.c

+1 -1

@@ -172,7 +172,7 @@ static DWORD netdata_windows_get_current_build()

172 cBuild, 63, HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", "CurrentBuild"))

173 return 0;

174

175 - errno = 0;

175 + errno_clear();

176

177 DWORD version = strtol(cBuild, NULL, 10);

178 if (errno == ERANGE)

src/daemon/winsvc.cc

+1 -1

@@ -219,7 +219,7 @@ static bool update_path() {

219

220 int main(int argc, char *argv[])

221 {

222 - bool tty = isatty(fileno(stdout)) == 1;

222 + bool tty = isatty(fileno(stdin)) == 1;

223

224 if (!update_path()) {

225 return 1;

src/database/engine/rrdengine.c

+1 -1

@@ -1517,7 +1517,7 @@ static void *journal_v2_indexing_tp_worker(struct rrdengine_instance *ctx __mayb

1517 break;

1518 }

1519

1520 - errno = 0;

1520 + errno_clear();

1521 if(count)

1522 nd_log(NDLS_DAEMON, NDLP_DEBUG,

1523 "DBENGINE: journal indexing done; %u files processed",

src/database/rrd.h

+2 -1

@@ -1043,7 +1043,6 @@ struct alarm_entry {

1043 STRING *recipient;

1044 time_t exec_run_timestamp;

1045 int exec_code;

1046 - uint64_t exec_spawn_serial;

1046

1047 STRING *source;

1048 STRING *units;

@@ -1069,6 +1068,8 @@ struct alarm_entry {

1068

1069 time_t last_repeat;

1070

1071 + POPEN_INSTANCE *popen_instance;

1072 +

1073 struct alarm_entry *next;

1074 struct alarm_entry *next_in_progress;

1075 struct alarm_entry *prev_in_progress;

src/database/rrdhost.c

+4 -6

@@ -1494,18 +1494,16 @@ static void rrdhost_load_kubernetes_labels(void) {

1494 return;

1495 }

1496

1497 - pid_t pid;

1498 - FILE *fp_child_input;

1499 - FILE *fp_child_output = netdata_popen(label_script, &pid, &fp_child_input);

1500 - if(!fp_child_output) return;

1497 + POPEN_INSTANCE *instance = spawn_popen_run(label_script);

1498 + if(!instance) return;

1499

1500 char buffer[1000 + 1];

1503 - while (fgets(buffer, 1000, fp_child_output) != NULL)

1501 + while (fgets(buffer, 1000, instance->child_stdout_fp) != NULL)

1502 rrdlabels_add_pair(localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);

1503

1504 // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'

1505 // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null

1508 - int rc = netdata_pclose(fp_child_input, fp_child_output, pid);

1506 + int rc = spawn_popen_wait(instance);

1507 if(rc)

1508 nd_log(NDLS_DAEMON, NDLP_ERR,

1509 "%s exited abnormally. Failed to get kubernetes labels.",

src/database/sqlite/sqlite_context.c

+1 -1

@@ -43,7 +43,7 @@ int sql_init_context_database(int memory)

43 return 1;

44 }

46 - errno = 0;

46 + errno_clear();

47 netdata_log_info("SQLite database %s initialization", sqlite_database);

49 char buf[1024 + 1] = "";

src/database/sqlite/sqlite_db_migration.c

+1 -1

@@ -518,7 +518,7 @@ static int migrate_database(sqlite3 *database, int target_version, char *db_name

518 }

519

520 if (likely(user_version == target_version)) {

521 - errno = 0;

521 + errno_clear();

522 netdata_log_info("%s database version is %d (no migration needed)", db_name, target_version);

523 return target_version;

524 }

src/database/sqlite/sqlite_health.c

+1 -1

@@ -461,7 +461,7 @@ void sql_alert_cleanup(bool cli)

461 {

462 UNUSED(cli);

463

464 - errno = 0;

464 + errno_clear();

465 if (sql_init_meta_database(DB_CHECK_NONE, 0)) {

466 netdata_log_error("Failed to open database");

467 return;

src/database/sqlite/sqlite_metadata.c

+2 -2

@@ -711,7 +711,7 @@ int sql_init_meta_database(db_check_action_type_t rebuild, int memory)

711 }

712

713 if (rebuild & DB_CHECK_ANALYZE) {

714 - errno = 0;

714 + errno_clear();

715 netdata_log_info("Running ANALYZE on %s", sqlite_database);

716 rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg);

717 if (rc != SQLITE_OK) {

@@ -725,7 +725,7 @@ int sql_init_meta_database(db_check_action_type_t rebuild, int memory)

725 return 1;

726 }

727

728 - errno = 0;

728 + errno_clear();

729 netdata_log_info("SQLite database %s initialization", sqlite_database);

730

731 rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);

src/exporting/send_data.c

+1 -1

@@ -77,7 +77,7 @@ void simple_connector_receive_response(int *sock, struct instance *instance)

77 ERR_clear_error();

78 #endif

80 - errno = 0;

80 + errno_clear();

82 // loop through to collect all data

83 while (*sock != -1 && errno != EWOULDBLOCK) {

src/health/health_log.c

+1 -1

@@ -43,7 +43,7 @@ void health_log_alert_transition_with_trace(RRDHOST *host, ALARM_ENTRY *ae, int

43 };

44 ND_LOG_STACK_PUSH(lgs);

46 - errno = 0;

46 + errno_clear();

48 ND_LOG_FIELD_PRIORITY priority = NDLP_INFO;

src/health/health_notifications.c

+8 -3

@@ -23,7 +23,13 @@ void health_alarm_wait_for_execution(ALARM_ENTRY *ae) {

23 if (!(ae->flags & HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS))

24 return;

26 - spawn_wait_cmd(ae->exec_spawn_serial, &ae->exec_code, &ae->exec_run_timestamp);

26 + if(!ae->popen_instance) {

27 + // nd_log(NDLS_DAEMON, NDLP_ERR, "attempted to wait for the execution of alert that has not spawn a notification");

28 + return;

29 + }

30 +

31 + ae->exec_code = spawn_popen_wait(ae->popen_instance);

32 +

33 netdata_log_debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);

34 ae->flags &= ~HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS;

@@ -75,7 +81,6 @@ static inline void enqueue_alarm_notify_in_progress(ALARM_ENTRY *ae)

81 alarm_notifications_in_progress.head = ae;

82 }

83 alarm_notifications_in_progress.tail = ae;

78 -

84 }

86 static bool prepare_command(BUFFER *wb,

@@ -462,7 +467,7 @@ void health_send_notification(RRDHOST *host, ALARM_ENTRY *ae, struct health_rais

467

468 netdata_log_debug(D_HEALTH, "executing command '%s'", command_to_run);

469 ae->flags |= HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS;

465 - ae->exec_spawn_serial = spawn_enq_cmd(command_to_run);

470 + ae->popen_instance = spawn_popen_run(command_to_run);

471 enqueue_alarm_notify_in_progress(ae);

472 health_alarm_log_save(host, ae);

473 } else {

src/libnetdata/clocks/clocks.c

+2 -2

@@ -343,7 +343,7 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) {

343 }

344

345 if(unlikely(now < next)) {

346 - errno = 0;

346 + errno_clear();

347 nd_log_limit_static_global_var(erl, 10, 0);

348 nd_log_limit(&erl, NDLS_DAEMON, NDLP_NOTICE,

349 "heartbeat clock: woke up %"PRIu64" microseconds earlier than expected "

@@ -351,7 +351,7 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) {

351 next - now);

352 }

353 else if(unlikely(now - next > tick / 2)) {

354 - errno = 0;

354 + errno_clear();

355 nd_log_limit_static_global_var(erl, 10, 0);

356 nd_log_limit(&erl, NDLS_DAEMON, NDLP_NOTICE,

357 "heartbeat clock: woke up %"PRIu64" microseconds later than expected "

src/libnetdata/libnetdata.c

+12 -90

@@ -1248,7 +1248,7 @@ cleanup:

1248 close(fd);

1249 }

1250 if(mem == MAP_FAILED) return NULL;

1251 - errno = 0;

1251 + errno_clear();

1252 return mem;

1253 }

1254

@@ -1364,7 +1364,7 @@ int verify_netdata_host_prefix(bool log_msg) {

1364 char buffer[FILENAME_MAX + 1];

1365 char *path = netdata_configured_host_prefix;

1366 char *reason = "unknown reason";

1367 - errno = 0;

1367 + errno_clear();

1368

1369 struct stat sb;

1370 if (stat(path, &sb) == -1) {

@@ -1679,19 +1679,17 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c

1679 return value;

1680 }

1681

1682 -

1682 BUFFER *run_command_and_get_output_to_buffer(const char *command, int max_line_length) {

1683 BUFFER *wb = buffer_create(0, NULL);

1684

1686 - pid_t pid;

1687 - FILE *fp = netdata_popen(command, &pid, NULL);

1688 -

1689 - if(fp) {

1685 + POPEN_INSTANCE *pi = spawn_popen_run(command);

1686 + if(pi) {

1687 char buffer[max_line_length + 1];

1691 - while (fgets(buffer, max_line_length, fp)) {

1688 + while (fgets(buffer, max_line_length, pi->child_stdout_fp)) {

1689 buffer[max_line_length] = '\0';

1690 buffer_strcat(wb, buffer);

1691 }

1692 + spawn_popen_kill(pi);

1693 }

1694 else {

1695 buffer_free(wb);

@@ -1699,103 +1697,27 @@ BUFFER *run_command_and_get_output_to_buffer(const char *command, int max_line_l

1697 return NULL;

1698 }

1699

1702 - netdata_pclose(NULL, fp, pid);

1700 return wb;

1701 }

1702

1703 bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length) {

1707 - pid_t pid;

1708 - FILE *fp = netdata_popen(command, &pid, NULL);

1709 -

1710 - if(fp) {

1704 + POPEN_INSTANCE *pi = spawn_popen_run(command);

1705 + if(pi) {

1706 char buffer[max_line_length + 1];

1712 - while (fgets(buffer, max_line_length, fp))

1707 +

1708 + while (fgets(buffer, max_line_length, pi->child_stdout_fp))

1709 fprintf(stdout, "%s", buffer);

1710 +

1711 + spawn_popen_kill(pi);

1712 }

1713 else {

1714 netdata_log_error("Failed to execute command '%s'.", command);

1715 return false;

1716 }

1717

1720 - netdata_pclose(NULL, fp, pid);

1718 return true;

1719 }

1720

1724 -

1725 -static int fd_is_valid(int fd) {

1726 - return fcntl(fd, F_GETFD) != -1 || errno != EBADF;

1727 -}

1728 -

1729 -void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){

1730 - int fd;

1731 -

1732 - switch(action){

1733 - case OPEN_FD_ACTION_CLOSE:

1734 - if(!(excluded_fds & OPEN_FD_EXCLUDE_STDIN)) (void)close(STDIN_FILENO);

1735 - if(!(excluded_fds & OPEN_FD_EXCLUDE_STDOUT)) (void)close(STDOUT_FILENO);

1736 - if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)close(STDERR_FILENO);

1737 -#if defined(HAVE_CLOSE_RANGE)

1738 - if(close_range(STDERR_FILENO + 1, ~0U, 0) == 0) return;

1739 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "close_range() failed, will try to close fds one by one");

1740 -#endif

1741 - break;

1742 - case OPEN_FD_ACTION_FD_CLOEXEC:

1743 - if(!(excluded_fds & OPEN_FD_EXCLUDE_STDIN)) (void)fcntl(STDIN_FILENO, F_SETFD, FD_CLOEXEC);

1744 - if(!(excluded_fds & OPEN_FD_EXCLUDE_STDOUT)) (void)fcntl(STDOUT_FILENO, F_SETFD, FD_CLOEXEC);

1745 - if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)fcntl(STDERR_FILENO, F_SETFD, FD_CLOEXEC);

1746 -#if defined(HAVE_CLOSE_RANGE) && defined(CLOSE_RANGE_CLOEXEC) // Linux >= 5.11, FreeBSD >= 13.1

1747 - if(close_range(STDERR_FILENO + 1, ~0U, CLOSE_RANGE_CLOEXEC) == 0) return;

1748 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "close_range() failed, will try to mark fds for closing one by one");

1749 -#endif

1750 - break;

1751 - default:

1752 - break; // do nothing

1753 - }

1754 -

1755 - DIR *dir = opendir("/proc/self/fd");

1756 - if (dir == NULL) {

1757 - struct rlimit rl;

1758 - int open_max = -1;

1759 -

1760 - if(getrlimit(RLIMIT_NOFILE, &rl) == 0 && rl.rlim_max != RLIM_INFINITY) open_max = rl.rlim_max;

1761 -#ifdef _SC_OPEN_MAX

1762 - else open_max = sysconf(_SC_OPEN_MAX);

1763 -#endif

1764 -

1765 - if (open_max == -1) open_max = 65535; // 65535 arbitrary default if everything else fails

1766 -

1767 - for (fd = STDERR_FILENO + 1; fd < open_max; fd++) {

1768 - switch(action){

1769 - case OPEN_FD_ACTION_CLOSE:

1770 - if(fd_is_valid(fd)) (void)close(fd);

1771 - break;

1772 - case OPEN_FD_ACTION_FD_CLOEXEC:

1773 - (void)fcntl(fd, F_SETFD, FD_CLOEXEC);

1774 - break;

1775 - default:

1776 - break; // do nothing

1777 - }

1778 - }

1779 - } else {

1780 - struct dirent *entry;

1781 - while ((entry = readdir(dir)) != NULL) {

1782 - fd = str2i(entry->d_name);

1783 - if(unlikely((fd == STDIN_FILENO ) || (fd == STDOUT_FILENO) || (fd == STDERR_FILENO) )) continue;

1784 - switch(action){

1785 - case OPEN_FD_ACTION_CLOSE:

1786 - if(fd_is_valid(fd)) (void)close(fd);

1787 - break;

1788 - case OPEN_FD_ACTION_FD_CLOEXEC:

1789 - (void)fcntl(fd, F_SETFD, FD_CLOEXEC);

1790 - break;

1791 - default:

1792 - break; // do nothing

1793 - }

1794 - }

1795 - closedir(dir);

1796 - }

1797 -}

1798 -

1721 struct timing_steps {

1722 const char *name;

1723 usec_t time;

src/libnetdata/libnetdata.h

+7 -14

@@ -326,6 +326,9 @@ size_t judy_aral_structures(void);

326

327 #define GUID_LEN 36

328

329 +#define PIPE_READ 0

330 +#define PIPE_WRITE 1

331 +

332 #include "linked-lists.h"

333 #include "storage-point.h"

334

@@ -425,7 +428,7 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c

428 #define UNUSED_FUNCTION(x) UNUSED_##x

429 #endif

430

428 -#define error_report(x, args...) do { errno = 0; netdata_log_error(x, ##args); } while(0)

431 +#define error_report(x, args...) do { errno_clear(); netdata_log_error(x, ##args); } while(0)

432

433 // Taken from linux kernel

434 #define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))

@@ -440,17 +443,6 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c

443 bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length);

444 struct web_buffer *run_command_and_get_output_to_buffer(const char *command, int max_line_length);

445

443 -typedef enum {

444 - OPEN_FD_ACTION_CLOSE,

445 - OPEN_FD_ACTION_FD_CLOEXEC

446 -} OPEN_FD_ACTION;

447 -typedef enum {

448 - OPEN_FD_EXCLUDE_STDIN = 0x01,

449 - OPEN_FD_EXCLUDE_STDOUT = 0x02,

450 - OPEN_FD_EXCLUDE_STDERR = 0x04

451 -} OPEN_FD_EXCLUDE;

452 -void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds);

453 -

446 #ifdef OS_WINDOWS

447 void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data);

448 #else

@@ -483,7 +475,9 @@ extern char *netdata_configured_host_prefix;

475 #include "datetime/rfc3339.h"

476 #include "datetime/rfc7231.h"

477 #include "completion/completion.h"

486 -#include "popen/popen.h"

478 +#include "log/log.h"

479 +#include "spawn_server/spawn_server.h"

480 +#include "spawn_server/spawn_popen.h"

481 #include "simple_pattern/simple_pattern.h"

482 #ifdef ENABLE_HTTPS

483 # include "socket/security.h"

@@ -491,7 +485,6 @@ extern char *netdata_configured_host_prefix;

485 #include "socket/socket.h"

486 #include "config/appconfig.h"

487 #include "log/journal.h"

494 -#include "log/log.h"

488 #include "buffered_reader/buffered_reader.h"

489 #include "procfile/procfile.h"

490 #include "string/string.h"

src/libnetdata/log/log.c

+100 -6

@@ -6,6 +6,10 @@

7 #include "../libnetdata.h"

9 +#if defined(OS_WINDOWS)

10 +#include <windows.h>

11 +#endif

12 +

13 #ifdef __FreeBSD__

14 #include <sys/endian.h>

15 #endif

@@ -35,6 +39,16 @@ int aclklog_enabled = 0;

39 struct nd_log_source;

40 static bool nd_log_limit_reached(struct nd_log_source *source);

42 +// ----------------------------------------------------------------------------

43 +

44 +void errno_clear(void) {

45 + errno = 0;

46 +

47 +#if defined(OS_WINDOWS)

48 + SetLastError(ERROR_SUCCESS);

49 +#endif

50 +}

51 +

52 // ----------------------------------------------------------------------------

53 // logging method

@@ -514,6 +528,13 @@ int nd_log_health_fd(void) {

528 return STDERR_FILENO;

529 }

530

531 +int nd_log_collectors_fd(void) {

532 + if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1)

533 + return nd_log.sources[NDLS_COLLECTORS].fd;

534 +

535 + return STDERR_FILENO;

536 +}

537 +

538 void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) {

539 char buf[FILENAME_MAX + 100];

540 if(setting && *setting)

@@ -1011,6 +1032,10 @@ static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf);

1032 static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf);

1033 static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf);

1034

1035 +#if defined(OS_WINDOWS)

1036 +static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf);

1037 +#endif

1038 +

1039 // ----------------------------------------------------------------------------

1040

1041 typedef void (*annotator_t)(BUFFER *wb, const char *key, struct log_field *lf);

@@ -1058,6 +1083,13 @@ static __thread struct log_field thread_log_fields[_NDF_MAX] = {

1083 .logfmt = "errno",

1084 .logfmt_annotator = errno_annotator,

1085 },

1086 +#if defined(OS_WINDOWS)

1087 + [NDF_WINERROR] = {

1088 + .journal = "WINERROR",

1089 + .logfmt = "winerror",

1090 + .logfmt_annotator = winerror_annotator,

1091 + },

1092 +#endif

1093 [NDF_INVOCATION_ID] = {

1094 .journal = "INVOCATION_ID", // standard journald field

1095 .logfmt = NULL,

@@ -1563,6 +1595,45 @@ static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf) {

1595 buffer_fast_strcat(wb, "\"", 1);

1596 }

1597

1598 +#if defined(OS_WINDOWS)

1599 +static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf) {

1600 + DWORD errnum = log_field_to_uint64(lf);

1601 +

1602 + if(errnum == 0)

1603 + return;

1604 +

1605 + char buf[1024];

1606 + DWORD size = FormatMessageA(

1607 + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,

1608 + NULL,

1609 + errnum,

1610 + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),

1611 + buf,

1612 + (DWORD)(sizeof(buf) - 1),

1613 + NULL

1614 + );

1615 + if(size > 0) {

1616 + // remove \r\n at the end

1617 + while(size > 0 && (buf[size - 1] == '\r' || buf[size - 1] == '\n'))

1618 + buf[--size] = '\0';

1619 + }

1620 + else

1621 + size = snprintf(buf, sizeof(buf) - 1, "unknown error code");

1622 +

1623 + buf[size] = '\0';

1624 +

1625 + if(buffer_strlen(wb))

1626 + buffer_fast_strcat(wb, " ", 1);

1627 +

1628 + buffer_strcat(wb, key);

1629 + buffer_fast_strcat(wb, "=\"", 2);

1630 + buffer_print_int64(wb, errnum);

1631 + buffer_fast_strcat(wb, ", ", 2);

1632 + buffer_json_strcat(wb, buf);

1633 + buffer_fast_strcat(wb, "\"", 1);

1634 +}

1635 +#endif

1636 +

1637 static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf) {

1638 uint64_t pri = log_field_to_uint64(lf);

1639

@@ -2099,8 +2170,8 @@ static void nd_logger_merge_log_stack_to_thread_fields(void) {

2170 }

2171

2172 static void nd_logger(const char *file, const char *function, const unsigned long line,

2102 - ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit, int saved_errno,

2103 - const char *fmt, va_list ap) {

2173 + ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit,

2174 + int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) {

2175

2176 SPINLOCK *spinlock;

2177 FILE *fp;

@@ -2168,6 +2239,11 @@ static void nd_logger(const char *file, const char *function, const unsigned lon

2239 if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set)

2240 thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno);

2241

2242 +#if defined(OS_WINDOWS)

2243 + if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set)

2244 + thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror);

2245 +#endif

2246 +

2247 CLEAN_BUFFER *wb = NULL;

2248 if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) {

2249 wb = buffer_create(1024, NULL);

@@ -2215,7 +2291,7 @@ static void nd_logger(const char *file, const char *function, const unsigned lon

2291 nd_log.sources[source].pending_msg = NULL;

2292 }

2293

2218 - errno = 0;

2294 + errno_clear();

2295 }

2296

2297 static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {

@@ -2234,6 +2310,12 @@ static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {

2310 void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... )

2311 {

2312 int saved_errno = errno;

2313 +

2314 + size_t saved_winerror = 0;

2315 +#if defined(OS_WINDOWS)

2316 + saved_winerror = GetLastError();

2317 +#endif

2318 +

2319 source = nd_log_validate_source(source);

2320

2321 if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)

@@ -2243,12 +2325,18 @@ void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const

2325 va_start(args, fmt);

2326 nd_logger(file, function, line, source, priority,

2327 source == NDLS_DAEMON || source == NDLS_COLLECTORS,

2246 - saved_errno, fmt, args);

2328 + saved_errno, saved_winerror, fmt, args);

2329 va_end(args);

2330 }

2331

2332 void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {

2333 int saved_errno = errno;

2334 +

2335 + size_t saved_winerror = 0;

2336 +#if defined(OS_WINDOWS)

2337 + saved_winerror = GetLastError();

2338 +#endif

2339 +

2340 source = nd_log_validate_source(source);

2341

2342 if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)

@@ -2272,7 +2360,7 @@ void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_F

2360 va_start(args, fmt);

2361 nd_logger(file, function, line, source, priority,

2362 source == NDLS_DAEMON || source == NDLS_COLLECTORS,

2275 - saved_errno, fmt, args);

2363 + saved_errno, saved_winerror, fmt, args);

2364 va_end(args);

2365 erl->last_logged = now;

2366 erl->count = 0;

@@ -2280,12 +2368,18 @@ void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_F

2368

2369 void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {

2370 int saved_errno = errno;

2371 +

2372 + size_t saved_winerror = 0;

2373 +#if defined(OS_WINDOWS)

2374 + saved_winerror = GetLastError();

2375 +#endif

2376 +

2377 ND_LOG_SOURCES source = NDLS_DAEMON;

2378 source = nd_log_validate_source(source);

2379

2380 va_list args;

2381 va_start(args, fmt);

2288 - nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, fmt, args);

2382 + nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args);

2383 va_end(args);

2384

2385 char date[LOG_DATE_LENGTH];

src/libnetdata/log/log.h

@@ -46,6 +46,9 @@ typedef enum __attribute__((__packed__)) {

46 NDF_LOG_SOURCE, // DAEMON, COLLECTORS, HEALTH, ACCESS, ACLK - set at the log call

47 NDF_PRIORITY, // the syslog priority (severity) - set at the log call

48 NDF_ERRNO, // the ERRNO at the time of the log call - added automatically

49 +#if defined(OS_WINDOWS)

50 + NDF_WINERROR, // Windows GetLastError()

51 +#endif

52 NDF_INVOCATION_ID, // the INVOCATION_ID of Netdata - added automatically

53 NDF_LINE, // the source code file line number - added automatically

54 NDF_FILE, // the source code filename - added automatically

@@ -141,6 +144,7 @@ typedef enum __attribute__((__packed__)) {

144 NDFT_CALLBACK,

145 } ND_LOG_STACK_FIELD_TYPE;

146

147 +void errno_clear(void);

148 void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting);

149 void nd_log_set_facility(const char *facility);

150 void nd_log_set_priority_level(const char *setting);

@@ -157,6 +161,7 @@ const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);

161 const char *nd_log_method_for_external_plugins(const char *s);

162

163 int nd_log_health_fd(void);

164 +int nd_log_collectors_fd(void);

165 typedef bool (*log_formatter_callback_t)(BUFFER *wb, void *data);

166

167 struct log_stack_entry {

src/libnetdata/maps/local-sockets.h

+400 -264

@@ -5,10 +5,8 @@

6 #include "libnetdata/libnetdata.h"

8 -// disable libmnl for the moment

9 -#undef HAVE_LIBMNL

10 -

8 #ifdef HAVE_LIBMNL

9 +#include <linux/rtnetlink.h>

10 #include <linux/inet_diag.h>

11 #include <linux/sock_diag.h>

12 #include <linux/unix_diag.h>

@@ -67,30 +65,41 @@ struct local_port;

65 struct local_socket_state;

66 typedef void (*local_sockets_cb_t)(struct local_socket_state *state, struct local_socket *n, void *data);

68 +struct local_sockets_config {

69 + bool listening;

70 + bool inbound;

71 + bool outbound;

72 + bool local;

73 + bool tcp4;

74 + bool tcp6;

75 + bool udp4;

76 + bool udp6;

77 + bool pid;

78 + bool cmdline;

79 + bool comm;

80 + bool uid;

81 + bool namespaces;

82 + bool tcp_info;

83 +

84 + size_t max_errors;

85 + size_t max_concurrent_namespaces;

86 +

87 + local_sockets_cb_t cb;

88 + void *data;

89 +

90 + const char *host_prefix;

91 +

92 + // internal use

93 + uint64_t net_ns_inode;

94 +};

95 +

96 typedef struct local_socket_state {

71 - struct {

72 - bool listening;

73 - bool inbound;

74 - bool outbound;

75 - bool local;

76 - bool tcp4;

77 - bool tcp6;

78 - bool udp4;

79 - bool udp6;

80 - bool pid;

81 - bool cmdline;

82 - bool comm;

83 - bool uid;

84 - bool namespaces;

85 - size_t max_errors;

86 -

87 - local_sockets_cb_t cb;

88 - void *data;

89 -

90 - const char *host_prefix;

91 - } config;

97 + struct local_sockets_config config;

99 struct {

100 + size_t mnl_sends;

101 + size_t namespaces_found;

102 + size_t tcp_info_received;

103 size_t pid_fds_processed;

104 size_t pid_fds_opendir_failed;

105 size_t pid_fds_readlink_failed;

@@ -98,6 +107,9 @@ typedef struct local_socket_state {

107 size_t errors_encountered;

108 } stats;

109

110 + bool spawn_server_is_mine;

111 + SPAWN_SERVER *spawn_server;

112 +

113 #ifdef HAVE_LIBMNL

114 bool use_nl;

115 struct mnl_socket *nl;

@@ -106,6 +118,7 @@ typedef struct local_socket_state {

118

119 ARAL *local_socket_aral;

120 ARAL *pid_socket_aral;

121 + SPINLOCK spinlock; // for namespaces

122

123 uint64_t proc_self_net_ns_inode;

124

@@ -181,12 +194,21 @@ typedef struct local_socket {

194 SOCKET_DIRECTION direction;

195

196 uint8_t timer;

184 - uint8_t retransmits;

197 + uint8_t retransmits; // the # of packets currently queued for retransmission (not yet acknowledged)

198 uint32_t expires;

199 uint32_t rqueue;

200 uint32_t wqueue;

201 uid_t uid;

202

203 + struct {

204 + bool checked;

205 + bool ipv46;

206 + } ipv6ony;

207 +

208 + union {

209 + struct tcp_info tcp;

210 + } info;

211 +

212 char comm[TASK_COMM_LEN];

213 STRING *cmdline;

214

@@ -201,16 +223,18 @@ typedef struct local_socket {

223 #endif

224 } LOCAL_SOCKET;

225

226 +static inline void local_sockets_spawn_server_callback(SPAWN_REQUEST *request);

227 +

228 // --------------------------------------------------------------------------------------------------------------------

229

230 static inline void local_sockets_log(LS_STATE *ls, const char *format, ...) PRINTFLIKE(2, 3);

231 static inline void local_sockets_log(LS_STATE *ls, const char *format, ...) {

208 - if(++ls->stats.errors_encountered == ls->config.max_errors) {

232 + if(ls && ++ls->stats.errors_encountered == ls->config.max_errors) {

233 nd_log(NDLS_COLLECTORS, NDLP_ERR, "LOCAL-SOCKETS: max number of logs reached. Not logging anymore");

234 return;

235 }

236

213 - if(ls->stats.errors_encountered > ls->config.max_errors)

237 + if(ls && ls->stats.errors_encountered > ls->config.max_errors)

238 return;

239

240 char buf[16384];

@@ -224,6 +248,133 @@ static inline void local_sockets_log(LS_STATE *ls, const char *format, ...) {

248

249 // --------------------------------------------------------------------------------------------------------------------

250

251 +static bool local_sockets_is_ipv4_mapped_ipv6_address(const struct in6_addr *addr) {

252 + // An IPv4-mapped IPv6 address starts with 80 bits of zeros followed by 16 bits of ones

253 + static const unsigned char ipv4_mapped_prefix[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF };

254 + return memcmp(addr->s6_addr, ipv4_mapped_prefix, 12) == 0;

255 +}

256 +

257 +static bool local_sockets_is_loopback_address(struct socket_endpoint *se) {

258 + if (se->family == AF_INET) {

259 + // For IPv4, loopback addresses are in the 127.0.0.0/8 range

260 + return (ntohl(se->ip.ipv4) >> 24) == 127; // Check if the first byte is 127

261 + } else if (se->family == AF_INET6) {

262 + // Check if the address is an IPv4-mapped IPv6 address

263 + if (local_sockets_is_ipv4_mapped_ipv6_address(&se->ip.ipv6)) {

264 + // Extract the last 32 bits (IPv4 address) and check if it's in the 127.0.0.0/8 range

265 + uint8_t *ip6 = (uint8_t *)&se->ip.ipv6;

266 + const uint32_t ipv4_addr = *((const uint32_t *)(ip6 + 12));

267 + return (ntohl(ipv4_addr) >> 24) == 127;

268 + }

269 +

270 + // For IPv6, loopback address is ::1

271 + return memcmp(&se->ip.ipv6, &in6addr_loopback, sizeof(se->ip.ipv6)) == 0;

272 + }

273 +

274 + return false;

275 +}

276 +

277 +static inline bool local_sockets_is_ipv4_reserved_address(uint32_t ip) {

278 + // Check for the reserved address ranges

279 + ip = ntohl(ip);

280 + return (

281 + (ip >> 24 == 10) || // Private IP range (A class)

282 + (ip >> 20 == (172 << 4) + 1) || // Private IP range (B class)

283 + (ip >> 16 == (192 << 8) + 168) || // Private IP range (C class)

284 + (ip >> 24 == 127) || // Loopback address (127.0.0.0)

285 + (ip >> 24 == 0) || // Reserved (0.0.0.0)

286 + (ip >> 24 == 169 && (ip >> 16) == 254) || // Link-local address (169.254.0.0)

287 + (ip >> 16 == (192 << 8) + 0) // Test-Net (192.0.0.0)

288 + );

289 +}

290 +

291 +static inline bool local_sockets_is_private_address(struct socket_endpoint *se) {

292 + if (se->family == AF_INET) {

293 + return local_sockets_is_ipv4_reserved_address(se->ip.ipv4);

294 + }

295 + else if (se->family == AF_INET6) {

296 + uint8_t *ip6 = (uint8_t *)&se->ip.ipv6;

297 +

298 + // Check if the address is an IPv4-mapped IPv6 address

299 + if (local_sockets_is_ipv4_mapped_ipv6_address(&se->ip.ipv6)) {

300 + // Extract the last 32 bits (IPv4 address) and check if it's in the 127.0.0.0/8 range

301 + const uint32_t ipv4_addr = *((const uint32_t *)(ip6 + 12));

302 + return local_sockets_is_ipv4_reserved_address(ipv4_addr);

303 + }

304 +

305 + // Check for link-local addresses (fe80::/10)

306 + if ((ip6[0] == 0xFE) && ((ip6[1] & 0xC0) == 0x80))

307 + return true;

308 +

309 + // Check for Unique Local Addresses (ULA) (fc00::/7)

310 + if ((ip6[0] & 0xFE) == 0xFC)

311 + return true;

312 +

313 + // Check for multicast addresses (ff00::/8)

314 + if (ip6[0] == 0xFF)

315 + return true;

316 +

317 + // For IPv6, loopback address is :: or ::1

318 + return memcmp(&se->ip.ipv6, &in6addr_any, sizeof(se->ip.ipv6)) == 0 ||

319 + memcmp(&se->ip.ipv6, &in6addr_loopback, sizeof(se->ip.ipv6)) == 0;

320 + }

321 +

322 + return false;

323 +}

324 +

325 +static bool local_sockets_is_multicast_address(struct socket_endpoint *se) {

326 + if (se->family == AF_INET) {

327 + // For IPv4, check if the address is 0.0.0.0

328 + uint32_t ip = htonl(se->ip.ipv4);

329 + return (ip >= 0xE0000000 && ip <= 0xEFFFFFFF); // Multicast address range (224.0.0.0/4)

330 + }

331 + else if (se->family == AF_INET6) {

332 + // For IPv6, check if the address is ff00::/8

333 + uint8_t *ip6 = (uint8_t *)&se->ip.ipv6;

334 + return ip6[0] == 0xff;

335 + }

336 +

337 + return false;

338 +}

339 +

340 +static bool local_sockets_is_zero_address(struct socket_endpoint *se) {

341 + if (se->family == AF_INET) {

342 + // For IPv4, check if the address is 0.0.0.0

343 + return se->ip.ipv4 == 0;

344 + }

345 + else if (se->family == AF_INET6) {

346 + // For IPv6, check if the address is ::

347 + return memcmp(&se->ip.ipv6, &in6addr_any, sizeof(se->ip.ipv6)) == 0;

348 + }

349 +

350 + return false;

351 +}

352 +

353 +static inline const char *local_sockets_address_space(struct socket_endpoint *se) {

354 + if(local_sockets_is_zero_address(se))

355 + return "zero";

356 + else if(local_sockets_is_loopback_address(se))

357 + return "loopback";

358 + else if(local_sockets_is_multicast_address(se))

359 + return "multicast";

360 + else if(local_sockets_is_private_address(se))

361 + return "private";

362 + else

363 + return "public";

364 +}

365 +

366 +// --------------------------------------------------------------------------------------------------------------------

367 +

368 +static inline bool is_local_socket_ipv46(LOCAL_SOCKET *n) {

369 + return n->local.family == AF_INET6 &&

370 + n->direction == SOCKET_DIRECTION_LISTEN &&

371 + local_sockets_is_zero_address(&n->local) &&

372 + n->ipv6ony.checked &&

373 + n->ipv6ony.ipv46;

374 +}

375 +

376 +// --------------------------------------------------------------------------------------------------------------------

377 +

378 static void local_sockets_foreach_local_socket_call_cb(LS_STATE *ls) {

379 for(SIMPLE_HASHTABLE_SLOT_LOCAL_SOCKET *sl = simple_hashtable_first_read_only_LOCAL_SOCKET(&ls->sockets_hashtable);

380 sl;

@@ -425,123 +576,6 @@ static inline bool local_sockets_find_all_sockets_in_proc(LS_STATE *ls, const ch

576

577 // --------------------------------------------------------------------------------------------------------------------

578

428 -static bool local_sockets_is_ipv4_mapped_ipv6_address(const struct in6_addr *addr) {

429 - // An IPv4-mapped IPv6 address starts with 80 bits of zeros followed by 16 bits of ones

430 - static const unsigned char ipv4_mapped_prefix[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF };

431 - return memcmp(addr->s6_addr, ipv4_mapped_prefix, 12) == 0;

432 -}

433 -

434 -static bool local_sockets_is_loopback_address(struct socket_endpoint *se) {

435 - if (se->family == AF_INET) {

436 - // For IPv4, loopback addresses are in the 127.0.0.0/8 range

437 - return (ntohl(se->ip.ipv4) >> 24) == 127; // Check if the first byte is 127

438 - } else if (se->family == AF_INET6) {

439 - // Check if the address is an IPv4-mapped IPv6 address

440 - if (local_sockets_is_ipv4_mapped_ipv6_address(&se->ip.ipv6)) {

441 - // Extract the last 32 bits (IPv4 address) and check if it's in the 127.0.0.0/8 range

442 - uint8_t *ip6 = (uint8_t *)&se->ip.ipv6;

443 - const uint32_t ipv4_addr = *((const uint32_t *)(ip6 + 12));

444 - return (ntohl(ipv4_addr) >> 24) == 127;

445 - }

446 -

447 - // For IPv6, loopback address is ::1

448 - return memcmp(&se->ip.ipv6, &in6addr_loopback, sizeof(se->ip.ipv6)) == 0;

449 - }

450 -

451 - return false;

452 -}

453 -

454 -static inline bool local_sockets_is_ipv4_reserved_address(uint32_t ip) {

455 - // Check for the reserved address ranges

456 - ip = ntohl(ip);

457 - return (

458 - (ip >> 24 == 10) || // Private IP range (A class)

459 - (ip >> 20 == (172 << 4) + 1) || // Private IP range (B class)

460 - (ip >> 16 == (192 << 8) + 168) || // Private IP range (C class)

461 - (ip >> 24 == 127) || // Loopback address (127.0.0.0)

462 - (ip >> 24 == 0) || // Reserved (0.0.0.0)

463 - (ip >> 24 == 169 && (ip >> 16) == 254) || // Link-local address (169.254.0.0)

464 - (ip >> 16 == (192 << 8) + 0) // Test-Net (192.0.0.0)

465 - );

466 -}

467 -

468 -static inline bool local_sockets_is_private_address(struct socket_endpoint *se) {

469 - if (se->family == AF_INET) {

470 - return local_sockets_is_ipv4_reserved_address(se->ip.ipv4);

471 - }

472 - else if (se->family == AF_INET6) {

473 - uint8_t *ip6 = (uint8_t *)&se->ip.ipv6;

474 -

475 - // Check if the address is an IPv4-mapped IPv6 address

476 - if (local_sockets_is_ipv4_mapped_ipv6_address(&se->ip.ipv6)) {

477 - // Extract the last 32 bits (IPv4 address) and check if it's in the 127.0.0.0/8 range

478 - const uint32_t ipv4_addr = *((const uint32_t *)(ip6 + 12));

479 - return local_sockets_is_ipv4_reserved_address(ipv4_addr);

480 - }

481 -

482 - // Check for link-local addresses (fe80::/10)

483 - if ((ip6[0] == 0xFE) && ((ip6[1] & 0xC0) == 0x80))

484 - return true;

485 -

486 - // Check for Unique Local Addresses (ULA) (fc00::/7)

487 - if ((ip6[0] & 0xFE) == 0xFC)

488 - return true;

489 -

490 - // Check for multicast addresses (ff00::/8)

491 - if (ip6[0] == 0xFF)

492 - return true;

493 -

494 - // For IPv6, loopback address is :: or ::1

495 - return memcmp(&se->ip.ipv6, &in6addr_any, sizeof(se->ip.ipv6)) == 0 ||

496 - memcmp(&se->ip.ipv6, &in6addr_loopback, sizeof(se->ip.ipv6)) == 0;

497 - }

498 -

499 - return false;

500 -}

501 -

502 -static bool local_sockets_is_multicast_address(struct socket_endpoint *se) {

503 - if (se->family == AF_INET) {

504 - // For IPv4, check if the address is 0.0.0.0

505 - uint32_t ip = htonl(se->ip.ipv4);

506 - return (ip >= 0xE0000000 && ip <= 0xEFFFFFFF); // Multicast address range (224.0.0.0/4)

507 - }

508 - else if (se->family == AF_INET6) {

509 - // For IPv6, check if the address is ff00::/8

510 - uint8_t *ip6 = (uint8_t *)&se->ip.ipv6;

511 - return ip6[0] == 0xff;

512 - }

513 -

514 - return false;

515 -}

516 -

517 -static bool local_sockets_is_zero_address(struct socket_endpoint *se) {

518 - if (se->family == AF_INET) {

519 - // For IPv4, check if the address is 0.0.0.0

520 - return se->ip.ipv4 == 0;

521 - }

522 - else if (se->family == AF_INET6) {

523 - // For IPv6, check if the address is ::

524 - return memcmp(&se->ip.ipv6, &in6addr_any, sizeof(se->ip.ipv6)) == 0;

525 - }

526 -

527 - return false;

528 -}

529 -

530 -static inline const char *local_sockets_address_space(struct socket_endpoint *se) {

531 - if(local_sockets_is_zero_address(se))

532 - return "zero";

533 - else if(local_sockets_is_loopback_address(se))

534 - return "loopback";

535 - else if(local_sockets_is_multicast_address(se))

536 - return "multicast";

537 - else if(local_sockets_is_private_address(se))

538 - return "private";

539 - else

540 - return "public";

541 -}

542 -

543 -// --------------------------------------------------------------------------------------------------------------------

544 -

579 static inline void local_sockets_index_listening_port(LS_STATE *ls, LOCAL_SOCKET *n) {

580 if(n->direction & SOCKET_DIRECTION_LISTEN) {

581 // for the listening sockets, keep a hashtable with all the local ports

@@ -636,28 +670,31 @@ static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) {

670

671 #ifdef HAVE_LIBMNL

672

639 -static inline void local_sockets_netlink_init(LS_STATE *ls) {

640 - ls->use_nl = true;

673 +static inline void local_sockets_libmnl_init(LS_STATE *ls) {

674 ls->nl = mnl_socket_open(NETLINK_INET_DIAG);

642 - if (!ls->nl) {

643 - local_sockets_log(ls, "cannot open netlink socket");

675 + if (ls->nl == NULL) {

676 + local_sockets_log(ls, "cannot open libmnl netlink socket");

677 ls->use_nl = false;

678 }

646 -

647 - if (mnl_socket_bind(ls->nl, 0, MNL_SOCKET_AUTOPID) < 0) {

648 - local_sockets_log(ls, "cannot bind netlink socket");

679 + else if (mnl_socket_bind(ls->nl, 0, MNL_SOCKET_AUTOPID) < 0) {

680 + local_sockets_log(ls, "cannot bind libmnl netlink socket");

681 + mnl_socket_close(ls->nl);

682 + ls->nl = NULL;

683 ls->use_nl = false;

684 }

685 + else

686 + ls->use_nl = true;

687 }

688

653 -static inline void local_sockets_netlink_cleanup(LS_STATE *ls) {

689 +static inline void local_sockets_libmnl_cleanup(LS_STATE *ls) {

690 if(ls->nl) {

691 mnl_socket_close(ls->nl);

692 ls->nl = NULL;

693 + ls->use_nl = false;

694 }

695 }

696

660 -static inline int local_sockets_netlink_cb_data(const struct nlmsghdr *nlh, void *data) {

697 +static inline int local_sockets_libmnl_cb_data(const struct nlmsghdr *nlh, void *data) {

698 LS_STATE *ls = data;

699

700 struct inet_diag_msg *diag_msg = mnl_nlmsg_get_payload(nlh);

@@ -666,15 +703,19 @@ static inline int local_sockets_netlink_cb_data(const struct nlmsghdr *nlh, void

703 .inode = diag_msg->idiag_inode,

704 .direction = SOCKET_DIRECTION_NONE,

705 .state = diag_msg->idiag_state,

706 + .ipv6ony = {

707 + .checked = false,

708 + .ipv46 = false,

709 + },

710 .local = {

711 .protocol = ls->tmp_protocol,

712 .family = diag_msg->idiag_family,

672 - .port = diag_msg->id.idiag_sport,

713 + .port = ntohs(diag_msg->id.idiag_sport),

714 },

715 .remote = {

716 .protocol = ls->tmp_protocol,

717 .family = diag_msg->idiag_family,

677 - .port = diag_msg->id.idiag_dport,

718 + .port = ntohs(diag_msg->id.idiag_dport),

719 },

720 .timer = diag_msg->idiag_timer,

721 .retransmits = diag_msg->idiag_retrans,

@@ -693,12 +734,37 @@ static inline int local_sockets_netlink_cb_data(const struct nlmsghdr *nlh, void

734 memcpy(&n.remote.ip.ipv6, diag_msg->id.idiag_dst, sizeof(n.remote.ip.ipv6));

735 }

736

737 + struct rtattr *attr = (struct rtattr *)(diag_msg + 1);

738 + int rtattrlen = nlh->nlmsg_len - NLMSG_LENGTH(sizeof(*diag_msg));

739 + for (; !n.ipv6ony.checked && RTA_OK(attr, rtattrlen); attr = RTA_NEXT(attr, rtattrlen)) {

740 + switch (attr->rta_type) {

741 + case INET_DIAG_INFO: {

742 + if(ls->tmp_protocol == IPPROTO_TCP) {

743 + struct tcp_info *info = (struct tcp_info *)RTA_DATA(attr);

744 + n.info.tcp = *info;

745 + ls->stats.tcp_info_received++;

746 + }

747 + }

748 + break;

749 +

750 + case INET_DIAG_SKV6ONLY: {

751 + n.ipv6ony.checked = true;

752 + int ipv6only = *(int *)RTA_DATA(attr);

753 + n.ipv6ony.ipv46 = !ipv6only;

754 + }

755 + break;

756 +

757 + default:

758 + break;

759 + }

760 + }

761 +

762 local_sockets_add_socket(ls, &n);

763

764 return MNL_CB_OK;

765 }

766

701 -static inline bool local_sockets_netlink_get_sockets(LS_STATE *ls, uint16_t family, uint16_t protocol) {

767 +static inline bool local_sockets_libmnl_get_sockets(LS_STATE *ls, uint16_t family, uint16_t protocol) {

768 ls->tmp_protocol = protocol;

769

770 char buf[MNL_SOCKET_BUFFER_SIZE];

@@ -710,14 +776,22 @@ static inline bool local_sockets_netlink_get_sockets(LS_STATE *ls, uint16_t fami

776 req.sdiag_family = family;

777 req.sdiag_protocol = protocol;

778 req.idiag_states = -1;

779 + req.idiag_ext = 0;

780 +

781 + if(family == AF_INET6)

782 + req.idiag_ext |= 1 << (INET_DIAG_SKV6ONLY - 1);

783 +

784 + if(protocol == IPPROTO_TCP && ls->config.tcp_info)

785 + req.idiag_ext |= 1 << (INET_DIAG_INFO - 1);

786

787 nlh = mnl_nlmsg_put_header(buf);

788 nlh->nlmsg_type = SOCK_DIAG_BY_FAMILY;

716 - nlh->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST;

789 + nlh->nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST;

790 nlh->nlmsg_seq = seq = time(NULL);

791 mnl_nlmsg_put_extra_header(nlh, sizeof(req));

792 memcpy(mnl_nlmsg_get_payload(nlh), &req, sizeof(req));

793

794 + ls->stats.mnl_sends++;

795 if (mnl_socket_sendto(ls->nl, nlh, nlh->nlmsg_len) < 0) {

796 local_sockets_log(ls, "mnl_socket_send failed");

797 return false;

@@ -725,7 +799,7 @@ static inline bool local_sockets_netlink_get_sockets(LS_STATE *ls, uint16_t fami

799

800 ssize_t ret;

801 while ((ret = mnl_socket_recvfrom(ls->nl, buf, sizeof(buf))) > 0) {

728 - ret = mnl_cb_run(buf, ret, seq, portid, local_sockets_netlink_cb_data, ls);

802 + ret = mnl_cb_run(buf, ret, seq, portid, local_sockets_libmnl_cb_data, ls);

803 if (ret <= MNL_CB_STOP)

804 break;

805 }

@@ -774,6 +848,10 @@ static inline bool local_sockets_read_proc_net_x(LS_STATE *ls, const char *filen

848

849 LOCAL_SOCKET n = {

850 .direction = SOCKET_DIRECTION_NONE,

851 + .ipv6ony = {

852 + .checked = false,

853 + .ipv46 = false,

854 + },

855 .local = {

856 .family = family,

857 .protocol = protocol,

@@ -904,6 +982,10 @@ static inline void local_sockets_detect_directions(LS_STATE *ls) {

982 // --------------------------------------------------------------------------------------------------------------------

983

984 static inline void local_sockets_init(LS_STATE *ls) {

985 + ls->config.host_prefix = netdata_configured_host_prefix;

986 +

987 + spinlock_init(&ls->spinlock);

988 +

989 simple_hashtable_init_NET_NS(&ls->ns_hashtable, 1024);

990 simple_hashtable_init_PID_SOCKET(&ls->pid_sockets_hashtable, 65535);

991 simple_hashtable_init_LOCAL_SOCKET(&ls->sockets_hashtable, 65535);

@@ -923,9 +1005,36 @@ static inline void local_sockets_init(LS_STATE *ls) {

1005 65536,

1006 65536,

1007 NULL, NULL, NULL, false, true);

1008 +

1009 + memset(&ls->stats, 0, sizeof(ls->stats));

1010 +

1011 +#ifdef HAVE_LIBMNL

1012 + ls->use_nl = false;

1013 + ls->nl = NULL;

1014 + ls->tmp_protocol = 0;

1015 + local_sockets_libmnl_init(ls);

1016 +#endif

1017 +

1018 + if(ls->config.namespaces && ls->spawn_server == NULL) {

1019 + ls->spawn_server = spawn_server_create(NULL, local_sockets_spawn_server_callback, 0, NULL);

1020 + ls->spawn_server_is_mine = true;

1021 + }

1022 + else

1023 + ls->spawn_server_is_mine = false;

1024 }

1025

1026 static inline void local_sockets_cleanup(LS_STATE *ls) {

1027 +

1028 + if(ls->spawn_server_is_mine) {

1029 + spawn_server_destroy(ls->spawn_server);

1030 + ls->spawn_server = NULL;

1031 + ls->spawn_server_is_mine = false;

1032 + }

1033 +

1034 +#ifdef HAVE_LIBMNL

1035 + local_sockets_libmnl_cleanup(ls);

1036 +#endif

1037 +

1038 // free the sockets hashtable data

1039 for(SIMPLE_HASHTABLE_SLOT_LOCAL_SOCKET *sl = simple_hashtable_first_read_only_LOCAL_SOCKET(&ls->sockets_hashtable);

1040 sl;

@@ -963,8 +1072,8 @@ static inline void local_sockets_cleanup(LS_STATE *ls) {

1072

1073 static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) {

1074 #ifdef HAVE_LIBMNL

966 - if(ls->use_nl) {

967 - ls->use_nl = local_sockets_netlink_get_sockets(ls, family, protocol);

1075 + if(ls->nl && ls->use_nl) {

1076 + ls->use_nl = local_sockets_libmnl_get_sockets(ls, family, protocol);

1077

1078 if(ls->use_nl)

1079 return;

@@ -974,7 +1083,7 @@ static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *fi

1083 local_sockets_read_proc_net_x(ls, filename, family, protocol);

1084 }

1085

977 -static inline void local_sockets_read_sockets_from_proc(LS_STATE *ls) {

1086 +static inline void local_sockets_read_all_system_sockets(LS_STATE *ls) {

1087 char path[FILENAME_MAX + 1];

1088

1089 if(ls->config.namespaces) {

@@ -1036,7 +1145,52 @@ static inline void local_sockets_send_to_parent(struct local_socket_state *ls __

1145 local_sockets_log(ls, "failed to write cmdline to pipe");

1146 }

1147

1039 -static inline bool local_sockets_get_namespace_sockets(LS_STATE *ls, struct pid_socket *ps, pid_t *pid) {

1148 +static inline void local_sockets_spawn_server_callback(SPAWN_REQUEST *request) {

1149 + LS_STATE ls = { 0 };

1150 + ls.config = *((struct local_sockets_config *)request->data);

1151 +

1152 + // we don't need these inside namespaces

1153 + ls.config.cmdline = false;

1154 + ls.config.comm = false;

1155 + ls.config.pid = false;

1156 + ls.config.namespaces = false;

1157 +

1158 + // initialize local sockets

1159 + local_sockets_init(&ls);

1160 +

1161 + ls.config.host_prefix = ""; // we need the /proc of the container

1162 +

1163 + struct local_sockets_child_work cw = {

1164 + .net_ns_inode = ls.proc_self_net_ns_inode,

1165 + .fd = request->fds[1], // stdout

1166 + };

1167 +

1168 + ls.config.cb = local_sockets_send_to_parent;

1169 + ls.config.data = &cw;

1170 + ls.proc_self_net_ns_inode = ls.config.net_ns_inode;

1171 +

1172 + // switch namespace using the custom fd passed via the spawn server

1173 + if (setns(request->fds[3], CLONE_NEWNET) == -1) {

1174 + local_sockets_log(&ls, "failed to switch network namespace at child process using fd %d", request->fds[3]);

1175 + exit(EXIT_FAILURE);

1176 + }

1177 +

1178 + // read all sockets from /proc

1179 + local_sockets_read_all_system_sockets(&ls);

1180 +

1181 + // send all sockets to parent

1182 + local_sockets_foreach_local_socket_call_cb(&ls);

1183 +

1184 + // send the terminating socket

1185 + struct local_socket zero = {

1186 + .net_ns_inode = ls.config.net_ns_inode,

1187 + };

1188 + local_sockets_send_to_parent(&ls, &zero, &cw);

1189 +

1190 + exit(EXIT_SUCCESS);

1191 +}

1192 +

1193 +static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, struct pid_socket *ps) {

1194 char filename[1024];

1195 snprintfz(filename, sizeof(filename), "%s/proc/%d/ns/net", ls->config.host_prefix, ps->pid);

1196

@@ -1060,80 +1214,32 @@ static inline bool local_sockets_get_namespace_sockets(LS_STATE *ls, struct pid_

1214 return false;

1215 }

1216

1063 - int pipefd[2];

1064 - if (pipe(pipefd) != 0) {

1065 - local_sockets_log(ls, "cannot create pipe");

1217 + if(ls->spawn_server == NULL) {

1218 close(fd);

1219 + local_sockets_log(ls, "spawn server is not available");

1220 return false;

1221 }

1222

1070 - *pid = fork();

1071 - if (*pid == 0) {

1072 - // Child process

1073 - close(pipefd[0]);

1074 -

1075 - // local_sockets_log(ls, "child is here for inode %"PRIu64" and namespace %"PRIu64, ps->inode, ps->net_ns_inode);

1076 -

1077 - struct local_sockets_child_work cw = {

1078 - .net_ns_inode = ps->net_ns_inode,

1079 - .fd = pipefd[1],

1080 - };

1081 -

1082 - ls->config.host_prefix = ""; // we need the /proc of the container

1083 - ls->config.cb = local_sockets_send_to_parent;

1084 - ls->config.data = &cw;

1085 - ls->config.cmdline = false; // we have these already

1086 - ls->config.comm = false; // we have these already

1087 - ls->config.pid = false; // we have these already

1088 - ls->config.namespaces = false;

1089 - ls->proc_self_net_ns_inode = ps->net_ns_inode;

1090 -

1091 -

1092 - // switch namespace

1093 - if (setns(fd, CLONE_NEWNET) == -1) {

1094 - local_sockets_log(ls, "failed to switch network namespace at child process");

1095 - exit(EXIT_FAILURE);

1096 - }

1097 -

1098 -#ifdef HAVE_LIBMNL

1099 - local_sockets_netlink_cleanup(ls);

1100 - local_sockets_netlink_init(ls);

1101 -#endif

1102 -

1103 - // read all sockets from /proc

1104 - local_sockets_read_sockets_from_proc(ls);

1105 -

1106 - // send all sockets to parent

1107 - local_sockets_foreach_local_socket_call_cb(ls);

1223 + struct local_sockets_config config = ls->config;

1224 + config.net_ns_inode = ps->net_ns_inode;

1225 + SPAWN_INSTANCE *si = spawn_server_exec(ls->spawn_server, STDERR_FILENO, fd, NULL, &config, sizeof(config), SPAWN_INSTANCE_TYPE_CALLBACK);

1226 + close(fd); fd = -1;

1227

1109 - // send the terminating socket

1110 - struct local_socket zero = {

1111 - .net_ns_inode = ps->net_ns_inode,

1112 - };

1113 - local_sockets_send_to_parent(ls, &zero, &cw);

1114 -

1115 -#ifdef HAVE_LIBMNL

1116 - local_sockets_netlink_cleanup(ls);

1117 -#endif

1118 -

1119 - close(pipefd[1]); // Close write end of pipe

1120 - exit(EXIT_SUCCESS);

1228 + if(si == NULL) {

1229 + local_sockets_log(ls, "cannot create spawn instance");

1230 + return false;

1231 }

1122 - // parent

1123 -

1124 - close(fd);

1125 - close(pipefd[1]);

1232

1233 size_t received = 0;

1234 struct local_socket buf;

1129 - while(read(pipefd[0], &buf, sizeof(buf)) == sizeof(buf)) {

1235 + while(read(spawn_server_instance_read_fd(si), &buf, sizeof(buf)) == sizeof(buf)) {

1236 size_t len = 0;

1131 - if(read(pipefd[0], &len, sizeof(len)) != sizeof(len))

1237 + if(read(spawn_server_instance_read_fd(si), &len, sizeof(len)) != sizeof(len))

1238 local_sockets_log(ls, "failed to read cmdline length from pipe");

1239

1240 if(len) {

1241 char cmdline[len + 1];

1136 - if(read(pipefd[0], cmdline, len) != (ssize_t)len)

1242 + if(read(spawn_server_instance_read_fd(si), cmdline, len) != (ssize_t)len)

1243 local_sockets_log(ls, "failed to read cmdline from pipe");

1244 else {

1245 cmdline[len] = '\0';

@@ -1153,15 +1259,15 @@ static inline bool local_sockets_get_namespace_sockets(LS_STATE *ls, struct pid_

1259 break;

1260 }

1261

1262 + spinlock_lock(&ls->spinlock);

1263 +

1264 SIMPLE_HASHTABLE_SLOT_LOCAL_SOCKET *sl = simple_hashtable_get_slot_LOCAL_SOCKET(&ls->sockets_hashtable, buf.inode, &buf, true);

1265 LOCAL_SOCKET *n = SIMPLE_HASHTABLE_SLOT_DATA(sl);

1266 if(n) {

1267 string_freez(buf.cmdline);

1160 -

1268 // local_sockets_log(ls,

1269 // "ns inode %" PRIu64" (comm: '%s', pid: %u, ns: %"PRIu64") already exists in hashtable (comm: '%s', pid: %u, ns: %"PRIu64") - ignoring duplicate",

1270 // buf.inode, buf.comm, buf.pid, buf.net_ns_inode, n->comm, n->pid, n->net_ns_inode);

1164 - continue;

1271 }

1272 else {

1273 n = aral_mallocz(ls->local_socket_aral);

@@ -1170,75 +1276,109 @@ static inline bool local_sockets_get_namespace_sockets(LS_STATE *ls, struct pid_

1276

1277 local_sockets_index_listening_port(ls, n);

1278 }

1173 - }

1279

1175 - close(pipefd[0]);

1280 + spinlock_unlock(&ls->spinlock);

1281 + }

1282

1283 + spawn_server_exec_kill(ls->spawn_server, si);

1284 return received > 0;

1285 }

1286

1180 -static inline void local_socket_waitpid(LS_STATE *ls, pid_t pid) {

1181 - if(!pid) return;

1287 +struct local_sockets_namespace_worker {

1288 + LS_STATE *ls;

1289 + uint64_t inode;

1290 +};

1291 +

1292 +static inline void *local_sockets_get_namespace_sockets(void *arg) {

1293 + struct local_sockets_namespace_worker *data = arg;

1294 + LS_STATE *ls = data->ls;

1295 + const uint64_t inode = data->inode;

1296 +

1297 + spinlock_lock(&ls->spinlock);

1298 +

1299 + // find a pid_socket that has this namespace

1300 + for(SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl_pid = simple_hashtable_first_read_only_PID_SOCKET(&ls->pid_sockets_hashtable) ;

1301 + sl_pid ;

1302 + sl_pid = simple_hashtable_next_read_only_PID_SOCKET(&ls->pid_sockets_hashtable, sl_pid)) {

1303 + struct pid_socket *ps = SIMPLE_HASHTABLE_SLOT_DATA(sl_pid);

1304 + if(!ps || ps->net_ns_inode != inode) continue;

1305

1183 - int status;

1184 - waitpid(pid, &status, 0);

1306 + // now we have a pid that has the same namespace inode

1307

1186 - if (WIFEXITED(status) && WEXITSTATUS(status) != 0)

1187 - local_sockets_log(ls, "Child exited with status %d", WEXITSTATUS(status));

1188 - else if (WIFSIGNALED(status))

1189 - local_sockets_log(ls, "Child terminated by signal %d", WTERMSIG(status));

1308 + spinlock_unlock(&ls->spinlock);

1309 + const bool worked = local_sockets_get_namespace_sockets_with_pid(ls, ps);

1310 + spinlock_lock(&ls->spinlock);

1311 +

1312 + if(worked)

1313 + break;

1314 + }

1315 +

1316 + spinlock_unlock(&ls->spinlock);

1317 +

1318 + return NULL;

1319 }

1320

1321 static inline void local_sockets_namespaces(LS_STATE *ls) {

1193 - pid_t children[5] = { 0 };

1194 - size_t last_child = 0;

1322 + size_t threads = ls->config.max_concurrent_namespaces;

1323 + if(threads == 0) threads = 5;

1324 + if(threads > 100) threads = 100;

1325 +

1326 + size_t last_thread = 0;

1327 + ND_THREAD *workers[threads];

1328 + struct local_sockets_namespace_worker workers_data[threads];

1329 + memset(workers, 0, sizeof(workers));

1330 + memset(workers_data, 0, sizeof(workers_data));

1331 +

1332 + spinlock_lock(&ls->spinlock);

1333

1334 for(SIMPLE_HASHTABLE_SLOT_NET_NS *sl = simple_hashtable_first_read_only_NET_NS(&ls->ns_hashtable);

1335 sl;

1336 sl = simple_hashtable_next_read_only_NET_NS(&ls->ns_hashtable, sl)) {

1199 - uint64_t inode = (uint64_t)SIMPLE_HASHTABLE_SLOT_DATA(sl);

1337 + const uint64_t inode = (uint64_t)SIMPLE_HASHTABLE_SLOT_DATA(sl);

1338

1339 if(inode == ls->proc_self_net_ns_inode)

1340 continue;

1341

1204 - // find a pid_socket that has this namespace

1205 - for(SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl_pid = simple_hashtable_first_read_only_PID_SOCKET(&ls->pid_sockets_hashtable) ;

1206 - sl_pid ;

1207 - sl_pid = simple_hashtable_next_read_only_PID_SOCKET(&ls->pid_sockets_hashtable, sl_pid)) {

1208 - struct pid_socket *ps = SIMPLE_HASHTABLE_SLOT_DATA(sl_pid);

1209 - if(!ps || ps->net_ns_inode != inode) continue;

1342 + spinlock_unlock(&ls->spinlock);

1343

1211 - if(++last_child >= 5)

1212 - last_child = 0;

1344 + ls->stats.namespaces_found++;

1345

1214 - local_socket_waitpid(ls, children[last_child]);

1215 - children[last_child] = 0;

1346 + if(workers[last_thread] != NULL) {

1347 + if(++last_thread >= threads)

1348 + last_thread = 0;

1349

1217 - // now we have a pid that has the same namespace inode

1218 - if(local_sockets_get_namespace_sockets(ls, ps, &children[last_child]))

1219 - break;

1350 + if(workers[last_thread]) {

1351 + nd_thread_join(workers[last_thread]);

1352 + workers[last_thread] = NULL;

1353 + }

1354 }

1355 +

1356 + workers_data[last_thread].ls = ls;

1357 + workers_data[last_thread].inode = inode;

1358 + workers[last_thread] = nd_thread_create(

1359 + "local-sockets-worker", NETDATA_THREAD_OPTION_JOINABLE,

1360 + local_sockets_get_namespace_sockets, &workers_data[last_thread]);

1361 +

1362 + spinlock_lock(&ls->spinlock);

1363 }

1364

1223 - for(size_t i = 0; i < 5 ;i++)

1224 - local_socket_waitpid(ls, children[i]);

1365 + spinlock_unlock(&ls->spinlock);

1366 +

1367 + // wait all the threads running

1368 + for(size_t i = 0; i < threads ;i++) {

1369 + if(workers[i])

1370 + nd_thread_join(workers[i]);

1371 + }

1372 }

1373

1374 // --------------------------------------------------------------------------------------------------------------------

1375

1376 static inline void local_sockets_process(LS_STATE *ls) {

1230 -

1231 -#ifdef HAVE_LIBMNL

1232 - local_sockets_netlink_init(ls);

1233 -#endif

1234 -

1235 - ls->config.host_prefix = netdata_configured_host_prefix;

1236 -

1377 // initialize our hashtables

1378 local_sockets_init(ls);

1379

1380 // read all sockets from /proc

1241 - local_sockets_read_sockets_from_proc(ls);

1381 + local_sockets_read_all_system_sockets(ls);

1382

1383 // check all socket namespaces

1384 if(ls->config.namespaces)

@@ -1253,10 +1393,6 @@ static inline void local_sockets_process(LS_STATE *ls) {

1393

1394 // free all memory

1395 local_sockets_cleanup(ls);

1256 -

1257 -#ifdef HAVE_LIBMNL

1258 - local_sockets_netlink_cleanup(ls);

1259 -#endif

1396 }

1397

1398 static inline void ipv6_address_to_txt(struct in6_addr *in6_addr, char *dst) {

src/libnetdata/os/close_range.c new

+87

@@ -0,0 +1,87 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "../libnetdata.h"

4 +

5 +static int fd_is_valid(int fd) {

6 + errno_clear();

7 + return fcntl(fd, F_GETFD) != -1 || errno != EBADF;

8 +}

9 +

10 +int os_get_fd_open_max(void) {

11 + static int fd_open_max = CLOSE_RANGE_FD_MAX;

12 +

13 + if(fd_open_max != CLOSE_RANGE_FD_MAX)

14 + return fd_open_max;

15 +

16 + if(fd_open_max == CLOSE_RANGE_FD_MAX || fd_open_max == -1) {

17 + struct rlimit rl;

18 + if (getrlimit(RLIMIT_NOFILE, &rl) == 0 && rl.rlim_max != RLIM_INFINITY)

19 + fd_open_max = rl.rlim_max;

20 + }

21 +

22 +#ifdef _SC_OPEN_MAX

23 + if(fd_open_max == CLOSE_RANGE_FD_MAX || fd_open_max == -1) {

24 + fd_open_max = sysconf(_SC_OPEN_MAX);

25 + }

26 +#endif

27 +

28 + if(fd_open_max == CLOSE_RANGE_FD_MAX || fd_open_max == -1) {

29 + // Arbitrary default if everything else fails

30 + fd_open_max = 65535;

31 + }

32 +

33 + return fd_open_max;

34 +}

35 +

36 +void os_close_range(int first, int last) {

37 +#if defined(HAVE_CLOSE_RANGE)

38 + if(close_range(first, last, 0) == 0) return;

39 +#endif

40 +

41 +#if defined(OS_LINUX)

42 + DIR *dir = opendir("/proc/self/fd");

43 + if (dir != NULL) {

44 + struct dirent *entry;

45 + while ((entry = readdir(dir)) != NULL) {

46 + int fd = str2i(entry->d_name);

47 + if (fd >= first && (last == CLOSE_RANGE_FD_MAX || fd <= last) && fd_is_valid(fd))

48 + (void)close(fd);

49 + }

50 + closedir(dir);

51 + return;

52 + }

53 +#endif

54 +

55 + // Fallback to looping through all file descriptors if necessary

56 + if (last == CLOSE_RANGE_FD_MAX)

57 + last = os_get_fd_open_max();

58 +

59 + for (int fd = first; fd <= last; fd++) {

60 + if (fd_is_valid(fd)) (void)close(fd);

61 + }

62 +}

63 +

64 +static int compare_ints(const void *a, const void *b) {

65 + int int_a = *((int*)a);

66 + int int_b = *((int*)b);

67 + return (int_a > int_b) - (int_a < int_b);

68 +}

69 +

70 +void os_close_all_non_std_open_fds_except(int fds[], size_t fds_num) {

71 + if (fds_num == 0 || fds == NULL) {

72 + os_close_range(STDERR_FILENO + 1, CLOSE_RANGE_FD_MAX);

73 + return;

74 + }

75 +

76 + qsort(fds, fds_num, sizeof(int), compare_ints);

77 +

78 + int start = STDERR_FILENO + 1;

79 + for (size_t i = 0; i < fds_num; i++) {

80 + if (fds[i] > start)

81 + os_close_range(start, fds[i] - 1);

82 +

83 + start = fds[i] + 1;

84 + }

85 +

86 + os_close_range(start, CLOSE_RANGE_FD_MAX);

87 +}

src/libnetdata/os/close_range.h new

+12

@@ -0,0 +1,12 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef CLOSE_RANGE_H

4 +#define CLOSE_RANGE_H

5 +

6 +#define CLOSE_RANGE_FD_MAX (int)(~0U)

7 +

8 +int os_get_fd_open_max(void);

9 +void os_close_range(int first, int last);

10 +void os_close_all_non_std_open_fds_except(int fds[], size_t fds_num);

11 +

12 +#endif //CLOSE_RANGE_H

src/libnetdata/os/get_pid_max.c

+29 -16

@@ -2,13 +2,27 @@

3 #include "../libnetdata.h"

5 -pid_t pid_max = 32768;

5 +pid_t pid_max = 4194304;

6 +

7 pid_t os_get_system_pid_max(void) {

8 + static bool read = false;

9 + if(read) return pid_max;

10 + read = true;

11 +

12 #if defined(OS_MACOS)

13 + int mib[2];

14 + int maxproc;

15 + size_t len = sizeof(maxproc);

16 +

17 + mib[0] = CTL_KERN;

18 + mib[1] = KERN_MAXPROC;

19 +

20 + if (sysctl(mib, 2, &maxproc, &len, NULL, 0) == -1) {

21 + pid_max = 99999; // Fallback value

22 + nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot find system max pid. Assuming %d.", pid_max);

23 + }

24 + else pid_max = (pid_t)maxproc;

9 - // As we currently do not know a solution to query pid_max from the os

10 - // we use the number defined in bsd/sys/proc_internal.h in XNU sources

11 - pid_max = 99999;

26 return pid_max;

28 #elif defined(OS_FREEBSD)

@@ -17,41 +31,40 @@ pid_t os_get_system_pid_max(void) {

32 if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) {

33 pid_max = 99999;

20 - netdata_log_error("Assuming system's maximum pid is %d.", pid_max);

21 - } else {

22 - pid_max = tmp_pid_max;

34 + nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot get system max pid. Assuming %d.", pid_max);

35 }

36 + else

37 + pid_max = tmp_pid_max;

39 return pid_max;

41 #elif defined(OS_LINUX)

29 - static char read = 0;

30 - if(unlikely(read)) return pid_max;

31 - read = 1;

32 -

43 char filename[FILENAME_MAX + 1];

44 snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix?netdata_configured_host_prefix:"");

46 unsigned long long max = 0;

47 if(read_single_number_file(filename, &max) != 0) {

38 - netdata_log_error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max);

48 + nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max);

49 return pid_max;

50 }

52 if(!max) {

43 - netdata_log_error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max);

53 + nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max);

54 return pid_max;

55 }

57 pid_max = (pid_t) max;

58 return pid_max;

50 -#else

60 +#elif defined(OS_WINDOWS)

52 - // just a big default

62 + pid_max = (pid_t)0x7FFFFFFF;

63 + return pid_max;

64 +

65 +#else

54 - pid_max = 4194304;

67 + // return the default

68 return pid_max;

70 #endif

src/libnetdata/os/os.h

+2 -1

@@ -7,12 +7,13 @@

7 #include <sys/syscall.h>

8 #endif

10 +#include "setproctitle.h"

11 +#include "close_range.h"

12 #include "setresuid.h"

13 #include "setresgid.h"

14 #include "getgrouplist.h"

15 #include "adjtimex.h"

16 #include "gettid.h"

15 -#include "waitid.h"

17 #include "get_pid_max.h"

18 #include "get_system_cpus.h"

19 #include "tinysleep.h"

src/libnetdata/os/setproctitle.c new

+31

@@ -0,0 +1,31 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "../libnetdata.h"

4 +#include "setproctitle.h"

5 +

6 +void os_setproctitle(const char *new_name, const int argc, const char **argv) {

7 +#ifdef HAVE_SYS_PRCTL_H

8 + // Set the process name (comm)

9 + prctl(PR_SET_NAME, new_name, 0, 0, 0);

10 +#endif

11 +

12 +#ifdef __FreeBSD__

13 + // Set the process name on FreeBSD

14 + setproctitle("%s", new_name);

15 +#endif

16 +

17 + if(argc && argv) {

18 + // replace with spaces all parameters found (except argv[0])

19 + for(int i = 1; i < argc ;i++) {

20 + char *s = (char *)&argv[i][0];

21 + while(*s != '\0') *s++ = ' ';

22 + }

23 +

24 + // overwrite argv[0]

25 + size_t len = strlen(new_name);

26 + const size_t argv0_len = strlen(argv[0]);

27 + strncpyz((char *)argv[0], new_name, MIN(len, argv0_len));

28 + while(len < argv0_len)

29 + ((char *)argv[0])[len++] = ' ';

30 + }

31 +}

src/libnetdata/os/setproctitle.h new

@@ -0,0 +1,8 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef SETPROCTITLE_H

4 +#define SETPROCTITLE_H

5 +

6 +void os_setproctitle(const char *new_name, int argc, const char **argv);

7 +

8 +#endif //SETPROCTITLE_H

src/libnetdata/os/waitid.c deleted

-72

@@ -1,72 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include "../libnetdata.h"

4 -

5 -int os_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) {

6 -#if defined(HAVE_WAITID)

7 - return waitid(idtype, id, infop, options);

8 -#else

9 - // emulate waitid() using waitpid()

10 -

11 - // a cache for WNOWAIT

12 - static const struct pid_status empty = { 0, 0 };

13 - static __thread struct pid_status last = { 0, 0 }; // the cache

14 - struct pid_status current = { 0, 0 };

15 -

16 - // zero the infop structure

17 - memset(infop, 0, sizeof(*infop));

18 -

19 - // from the infop structure we use only 3 fields:

20 - // - si_pid

21 - // - si_code

22 - // - si_status

23 - // so, we update only these 3

24 -

25 - switch(idtype) {

26 - case P_ALL:

27 - current.pid = waitpid((pid_t)-1, &current.status, options);

28 - if(options & WNOWAIT)

29 - last = current;

30 - else

31 - last = empty;

32 - break;

33 -

34 - case P_PID:

35 - if(last.pid == (pid_t)id) {

36 - current = last;

37 - last = empty;

38 - }

39 - else

40 - current.pid = waitpid((pid_t)id, &current.status, options);

41 -

42 - break;

43 -

44 - default:

45 - errno = ENOSYS;

46 - return -1;

47 - }

48 -

49 - if (current.pid > 0) {

50 - if (WIFEXITED(current.status)) {

51 - infop->si_code = CLD_EXITED;

52 - infop->si_status = WEXITSTATUS(current.status);

53 - } else if (WIFSIGNALED(current.status)) {

54 - infop->si_code = WTERMSIG(current.status) == SIGABRT ? CLD_DUMPED : CLD_KILLED;

55 - infop->si_status = WTERMSIG(current.status);

56 - } else if (WIFSTOPPED(current.status)) {

57 - infop->si_code = CLD_STOPPED;

58 - infop->si_status = WSTOPSIG(current.status);

59 - } else if (WIFCONTINUED(current.status)) {

60 - infop->si_code = CLD_CONTINUED;

61 - infop->si_status = SIGCONT;

62 - }

63 - infop->si_pid = current.pid;

64 - return 0;

65 - } else if (current.pid == 0) {

66 - // No change in state, depends on WNOHANG

67 - return 0;

68 - }

69 -

70 - return -1;

71 -#endif

72 -}

src/libnetdata/os/waitid.h deleted

-48

@@ -1,48 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_WAITID_H

4 -#define NETDATA_WAITID_H

5 -

6 -#include "config.h"

7 -#include <sys/types.h>

8 -#include <signal.h>

9 -

10 -#ifdef HAVE_SYS_WAIT_H

11 -#include <sys/wait.h>

12 -#endif

13 -

14 -#ifndef WNOWAIT

15 -#define WNOWAIT 0x01000000

16 -#endif

17 -

18 -#ifndef WEXITED

19 -#define WEXITED 4

20 -#endif

21 -

22 -#if !defined(HAVE_WAITID)

23 -typedef enum

24 -{

25 - P_ALL, /* Wait for any child. */

26 - P_PID, /* Wait for specified process. */

27 - P_PGID, /* Wait for members of process group. */

28 - P_PIDFD, /* Wait for the child referred by the PID file descriptor. */

29 -} idtype_t;

30 -

31 -struct pid_status {

32 - pid_t pid;

33 - int status;

34 -};

35 -

36 -#if defined(OS_WINDOWS) && !defined(__CYGWIN__)

37 -typedef uint32_t id_t;

38 -typedef struct {

39 - int si_code; /* Signal code. */

40 - int si_status; /* Exit value or signal. */

41 - pid_t si_pid; /* Sending process ID. */

42 -} siginfo_t;

43 -#endif

44 -#endif

45 -

46 -int os_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options);

47 -

48 -#endif //NETDATA_WAITID_H

src/libnetdata/popen/README.md deleted

-15

@@ -1,15 +0,0 @@

1 -<!--

2 -title: "popen"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/popen/README.md

4 -sidebar_label: "popen"

5 -learn_status: "Published"

6 -learn_topic_type: "Tasks"

7 -learn_rel_path: "Developers/libnetdata"

8 --->

9 -

10 -# popen

11 -

12 -Process management library

13 -

14 -

15 -

src/libnetdata/popen/popen.c deleted

-446

@@ -1,446 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include "../libnetdata.h"

4 -

5 -// ----------------------------------------------------------------------------

6 -// popen with tracking

7 -

8 -static pthread_mutex_t netdata_popen_tracking_mutex = NETDATA_MUTEX_INITIALIZER;

9 -

10 -struct netdata_popen {

11 - pid_t pid;

12 - bool reaped;

13 - siginfo_t infop;

14 - int waitid_ret;

15 - struct netdata_popen *next;

16 - struct netdata_popen *prev;

17 -};

18 -

19 -static struct netdata_popen *netdata_popen_root = NULL;

20 -

21 -// myp_add_lock takes the lock if we're tracking.

22 -static void netdata_popen_tracking_lock(void) {

23 - netdata_mutex_lock(&netdata_popen_tracking_mutex);

24 -}

25 -

26 -// myp_add_unlock release the lock if we're tracking.

27 -static void netdata_popen_tracking_unlock(void) {

28 - netdata_mutex_unlock(&netdata_popen_tracking_mutex);

29 -}

30 -

31 -// myp_add_locked adds pid if we're tracking.

32 -// myp_add_lock must have been called previously.

33 -static void netdata_popen_tracking_add_pid_unsafe(pid_t pid) {

34 - struct netdata_popen *mp;

35 -

36 - mp = callocz(1, sizeof(struct netdata_popen));

37 - mp->pid = pid;

38 -

39 - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(netdata_popen_root, mp, prev, next);

40 -}

41 -

42 -// myp_del deletes pid if we're tracking.

43 -static void netdata_popen_tracking_del_pid(pid_t pid) {

44 - struct netdata_popen *mp;

45 -

46 - netdata_popen_tracking_lock();

47 -

48 - DOUBLE_LINKED_LIST_FOREACH_FORWARD(netdata_popen_root, mp, prev, next) {

49 - if(unlikely(mp->pid == pid))

50 - break;

51 - }

52 -

53 - if(mp) {

54 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(netdata_popen_root, mp, prev, next);

55 - freez(mp);

56 - }

57 - else

58 - netdata_log_error("POPEN: Cannot find pid %d.", pid);

59 -

60 - netdata_popen_tracking_unlock();

61 -}

62 -

63 -// myp_free cleans up any resources allocated for process

64 -// tracking.

65 -void netdata_popen_tracking_cleanup(void) {

66 - netdata_popen_tracking_lock();

67 -

68 - while(netdata_popen_root) {

69 - struct netdata_popen *mp = netdata_popen_root;

70 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(netdata_popen_root, mp, prev, next);

71 - freez(mp);

72 - }

73 -

74 - netdata_popen_tracking_unlock();

75 -}

76 -

77 -int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) {

78 - struct netdata_popen *mp = NULL;

79 -

80 - if(idtype == P_PID && id != 0) {

81 - // the caller is asking to waitid() for a specific child pid

82 -

83 - netdata_popen_tracking_lock();

84 - DOUBLE_LINKED_LIST_FOREACH_FORWARD(netdata_popen_root, mp, prev, next) {

85 - if(unlikely(mp->pid == (pid_t)id))

86 - break;

87 - }

88 -

89 - if(!mp)

90 - netdata_popen_tracking_unlock();

91 - }

92 -

93 - int ret;

94 - if(mp && mp->reaped) {

95 - // we have already reaped this child

96 - ret = mp->waitid_ret;

97 - *infop = mp->infop;

98 - }

99 - else {

100 - // we haven't reaped this child yet

101 - ret = os_waitid(idtype, id, infop, options);

102 -

103 - if(mp && !mp->reaped) {

104 - mp->reaped = true;

105 - mp->infop = *infop;

106 - mp->waitid_ret = ret;

107 - }

108 - }

109 -

110 - if(mp)

111 - netdata_popen_tracking_unlock();

112 -

113 - return ret;

114 -}

115 -

116 -// ----------------------------------------------------------------------------

117 -// helpers

118 -

119 -static inline void convert_argv_to_string(char *dst, size_t size, const char *spawn_argv[]) {

120 - int i;

121 - for(i = 0; spawn_argv[i] ;i++) {

122 - if(i == 0) snprintfz(dst, size, "%s", spawn_argv[i]);

123 - else {

124 - size_t len = strlen(dst);

125 - snprintfz(&dst[len], size - len, " '%s'", spawn_argv[i]);

126 - }

127 - }

128 -}

129 -

130 -// ----------------------------------------------------------------------------

131 -// the core of netdata popen

132 -

133 -/*

134 - * Returns -1 on failure, 0 on success. When POPEN_FLAG_CREATE_PIPE is set, on success set the FILE *fp pointer.

135 - */

136 -#define PIPE_READ 0

137 -#define PIPE_WRITE 1

138 -

139 -static int popene_internal(volatile pid_t *pidptr, char **env, uint8_t flags, FILE **fpp_child_stdin, FILE **fpp_child_stdout, const char *command, const char *spawn_argv[]) {

140 - // create a string to be logged about the command we are running

141 - char command_to_be_logged[2048];

142 - convert_argv_to_string(command_to_be_logged, sizeof(command_to_be_logged), spawn_argv);

143 - // netdata_log_info("custom_popene() running command: %s", command_to_be_logged);

144 -

145 - int ret = 0; // success by default

146 - int attr_rc = 1; // failure by default

147 -

148 - FILE *fp_child_stdin = NULL, *fp_child_stdout = NULL;

149 - int pipefd_stdin[2] = { -1, -1 };

150 - int pipefd_stdout[2] = { -1, -1 };

151 -

152 - pid_t pid;

153 - posix_spawnattr_t attr;

154 - posix_spawn_file_actions_t fa;

155 -

156 - unsigned int fds_to_exclude_from_closing = OPEN_FD_EXCLUDE_STDERR;

157 -

158 - if(posix_spawn_file_actions_init(&fa)) {

159 - netdata_log_error("POPEN: posix_spawn_file_actions_init() failed.");

160 - ret = -1;

161 - goto set_return_values_and_return;

162 - }

163 -

164 - if(fpp_child_stdin) {

165 - if (pipe(pipefd_stdin) == -1) {

166 - netdata_log_error("POPEN: stdin pipe() failed");

167 - ret = -1;

168 - goto cleanup_and_return;

169 - }

170 -

171 - if ((fp_child_stdin = fdopen(pipefd_stdin[PIPE_WRITE], "w")) == NULL) {

172 - netdata_log_error("POPEN: fdopen() stdin failed");

173 - ret = -1;

174 - goto cleanup_and_return;

175 - }

176 -

177 - if(posix_spawn_file_actions_adddup2(&fa, pipefd_stdin[PIPE_READ], STDIN_FILENO)) {

178 - netdata_log_error("POPEN: posix_spawn_file_actions_adddup2() on stdin failed.");

179 - ret = -1;

180 - goto cleanup_and_return;

181 - }

182 - }

183 - else {

184 - if (posix_spawn_file_actions_addopen(&fa, STDIN_FILENO, "/dev/null", O_RDONLY, 0)) {

185 - netdata_log_error("POPEN: posix_spawn_file_actions_addopen() on stdin to /dev/null failed.");

186 - // this is not a fatal error

187 - fds_to_exclude_from_closing |= OPEN_FD_EXCLUDE_STDIN;

188 - }

189 - }

190 -

191 - if (fpp_child_stdout) {

192 - if (pipe(pipefd_stdout) == -1) {

193 - netdata_log_error("POPEN: stdout pipe() failed");

194 - ret = -1;

195 - goto cleanup_and_return;

196 - }

197 -

198 - if ((fp_child_stdout = fdopen(pipefd_stdout[PIPE_READ], "r")) == NULL) {

199 - netdata_log_error("POPEN: fdopen() stdout failed");

200 - ret = -1;

201 - goto cleanup_and_return;

202 - }

203 -

204 - if(posix_spawn_file_actions_adddup2(&fa, pipefd_stdout[PIPE_WRITE], STDOUT_FILENO)) {

205 - netdata_log_error("POPEN: posix_spawn_file_actions_adddup2() on stdout failed.");

206 - ret = -1;

207 - goto cleanup_and_return;

208 - }

209 - }

210 - else {

211 - if (posix_spawn_file_actions_addopen(&fa, STDOUT_FILENO, "/dev/null", O_WRONLY, 0)) {

212 - netdata_log_error("POPEN: posix_spawn_file_actions_addopen() on stdout to /dev/null failed.");

213 - // this is not a fatal error

214 - fds_to_exclude_from_closing |= OPEN_FD_EXCLUDE_STDOUT;

215 - }

216 - }

217 -

218 - if(flags & POPEN_FLAG_CLOSE_FD) {

219 - // Mark all files to be closed by the exec() stage of posix_spawn()

220 - for_each_open_fd(OPEN_FD_ACTION_FD_CLOEXEC, fds_to_exclude_from_closing);

221 - }

222 -

223 - attr_rc = posix_spawnattr_init(&attr);

224 - if(attr_rc) {

225 - // failed

226 - netdata_log_error("POPEN: posix_spawnattr_init() failed.");

227 - }

228 - else {

229 - // success

230 - // reset all signals in the child

231 -

232 - if (posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF))

233 - netdata_log_error("POPEN: posix_spawnattr_setflags() failed.");

234 -

235 - sigset_t mask;

236 - sigemptyset(&mask);

237 -

238 - if (posix_spawnattr_setsigmask(&attr, &mask))

239 - netdata_log_error("POPEN: posix_spawnattr_setsigmask() failed.");

240 - }

241 -

242 - // Take the lock while we fork to ensure we don't race with SIGCHLD

243 - // delivery on a process which exits quickly.

244 - netdata_popen_tracking_lock();

245 - if (!posix_spawn(&pid, command, &fa, &attr, (char * const*)spawn_argv, env)) {

246 - // success

247 - *pidptr = pid;

248 - netdata_popen_tracking_add_pid_unsafe(pid);

249 - netdata_popen_tracking_unlock();

250 - }

251 - else {

252 - // failure

253 - netdata_popen_tracking_unlock();

254 - netdata_log_error("POPEN: failed to spawn command: \"%s\" from parent pid %d.", command_to_be_logged, getpid());

255 - ret = -1;

256 - goto cleanup_and_return;

257 - }

258 -

259 - // the normal cleanup will run

260 - // but ret == 0 at this point

261 -

262 -cleanup_and_return:

263 - if(!attr_rc) {

264 - // posix_spawnattr_init() succeeded

265 - if (posix_spawnattr_destroy(&attr))

266 - netdata_log_error("POPEN: posix_spawnattr_destroy() failed");

267 - }

268 -

269 - if (posix_spawn_file_actions_destroy(&fa))

270 - netdata_log_error("POPEN: posix_spawn_file_actions_destroy() failed");

271 -

272 - // the child end - close it

273 - if(pipefd_stdin[PIPE_READ] != -1)

274 - close(pipefd_stdin[PIPE_READ]);

275 -

276 - // our end

277 - if(ret == -1 || !fpp_child_stdin) {

278 - if (fp_child_stdin)

279 - fclose(fp_child_stdin);

280 - else if (pipefd_stdin[PIPE_WRITE] != -1)

281 - close(pipefd_stdin[PIPE_WRITE]);

282 -

283 - fp_child_stdin = NULL;

284 - }

285 -

286 - // the child end - close it

287 - if (pipefd_stdout[PIPE_WRITE] != -1)

288 - close(pipefd_stdout[PIPE_WRITE]);

289 -

290 - // our end

291 - if (ret == -1 || !fpp_child_stdout) {

292 - if (fp_child_stdout)

293 - fclose(fp_child_stdout);

294 - else if (pipefd_stdout[PIPE_READ] != -1)

295 - close(pipefd_stdout[PIPE_READ]);

296 -

297 - fp_child_stdout = NULL;

298 - }

299 -

300 -set_return_values_and_return:

301 - if(fpp_child_stdin)

302 - *fpp_child_stdin = fp_child_stdin;

303 -

304 - if(fpp_child_stdout)

305 - *fpp_child_stdout = fp_child_stdout;

306 -

307 - return ret;

308 -}

309 -

310 -int netdata_popene_variadic_internal_dont_use_directly(volatile pid_t *pidptr, char **env, uint8_t flags, FILE **fpp_child_input, FILE **fpp_child_output, const char *command, ...) {

311 - // convert the variable list arguments into what posix_spawn() needs

312 - // all arguments are expected strings

313 - va_list args;

314 - int args_count;

315 -

316 - // count the number variable parameters

317 - // the variable parameters are expected NULL terminated

318 - {

319 - const char *s;

320 -

321 - va_start(args, command);

322 - args_count = 0;

323 - while ((s = va_arg(args, const char *))) args_count++;

324 - va_end(args);

325 - }

326 -

327 - // create a string pointer array as needed by posix_spawn()

328 - // variable array in the stack

329 - const char *spawn_argv[args_count + 1];

330 - {

331 - const char *s;

332 - va_start(args, command);

333 - int i;

334 - for (i = 0; i < args_count; i++) {

335 - s = va_arg(args, const char *);

336 - spawn_argv[i] = s;

337 - }

338 - spawn_argv[args_count] = NULL;

339 - va_end(args);

340 - }

341 -

342 - return popene_internal(pidptr, env, flags, fpp_child_input, fpp_child_output, command, spawn_argv);

343 -}

344 -

345 -// See man environ

346 -extern char **environ;

347 -

348 -FILE *netdata_popen(const char *command, volatile pid_t *pidptr, FILE **fpp_child_input) {

349 - FILE *fp_child_output = NULL;

350 - const char *spawn_argv[] = {

351 - "sh",

352 - "-c",

353 - command,

354 - NULL

355 - };

356 - (void)popene_internal(pidptr, environ, POPEN_FLAG_CLOSE_FD, fpp_child_input, &fp_child_output, "/bin/sh", spawn_argv);

357 - return fp_child_output;

358 -}

359 -

360 -FILE *netdata_popene(const char *command, volatile pid_t *pidptr, char **env, FILE **fpp_child_input) {

361 - FILE *fp_child_output = NULL;

362 - const char *spawn_argv[] = {

363 - "sh",

364 - "-c",

365 - command,

366 - NULL

367 - };

368 - (void)popene_internal(pidptr, env, POPEN_FLAG_CLOSE_FD, fpp_child_input, &fp_child_output, "/bin/sh", spawn_argv);

369 - return fp_child_output;

370 -}

371 -

372 -// returns 0 on success, -1 on failure

373 -int netdata_spawn(const char *command, volatile pid_t *pidptr) {

374 - const char *spawn_argv[] = {

375 - "sh",

376 - "-c",

377 - command,

378 - NULL

379 - };

380 - return popene_internal(pidptr, environ, POPEN_FLAG_NONE, NULL, NULL, "/bin/sh", spawn_argv);

381 -}

382 -

383 -int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid) {

384 - int ret;

385 - siginfo_t info;

386 -

387 - netdata_log_debug(D_EXIT, "Request to netdata_pclose() on pid %d", pid);

388 -

389 - if (fp_child_input)

390 - fclose(fp_child_input);

391 -

392 - if (fp_child_output)

393 - fclose(fp_child_output);

394 -

395 - errno = 0;

396 -

397 - ret = netdata_waitid(P_PID, (id_t) pid, &info, WEXITED);

398 - netdata_popen_tracking_del_pid(pid);

399 -

400 - if (ret != -1) {

401 - switch (info.si_code) {

402 - case CLD_EXITED:

403 - if(info.si_status)

404 - netdata_log_error("child pid %d exited with code %d.", info.si_pid, info.si_status);

405 - return(info.si_status);

406 -

407 - case CLD_KILLED:

408 - if(info.si_status == SIGTERM) {

409 - netdata_log_info("child pid %d killed by SIGTERM", info.si_pid);

410 - return(0);

411 - }

412 - else if(info.si_status == SIGPIPE) {

413 - netdata_log_info("child pid %d killed by SIGPIPE.", info.si_pid);

414 - return(0);

415 - }

416 - else {

417 - netdata_log_error("child pid %d killed by signal %d.", info.si_pid, info.si_status);

418 - return(-1);

419 - }

420 -

421 - case CLD_DUMPED:

422 - netdata_log_error("child pid %d core dumped by signal %d.", info.si_pid, info.si_status);

423 - return(-2);

424 -

425 - case CLD_STOPPED:

426 - netdata_log_error("child pid %d stopped by signal %d.", info.si_pid, info.si_status);

427 - return(0);

428 -

429 - case CLD_TRAPPED:

430 - netdata_log_error("child pid %d trapped by signal %d.", info.si_pid, info.si_status);

431 - return(-4);

432 -

433 - case CLD_CONTINUED:

434 - netdata_log_error("child pid %d continued by signal %d.", info.si_pid, info.si_status);

435 - return(0);

436 -

437 - default:

438 - netdata_log_error("child pid %d gave us a SIGCHLD with code %d and status %d.", info.si_pid, info.si_code, info.si_status);

439 - return(-5);

440 - }

441 - }

442 - else

443 - netdata_log_error("Cannot waitid() for pid %d", pid);

444 -

445 - return 0;

446 -}

src/libnetdata/popen/popen.h deleted

-35

@@ -1,35 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_POPEN_H

4 -#define NETDATA_POPEN_H 1

5 -

6 -#include "../os/waitid.h"

7 -int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options);

8 -

9 -#include "../libnetdata.h"

10 -

11 -#define PIPE_READ 0

12 -#define PIPE_WRITE 1

13 -

14 -/* custom_popene_variadic_internal_dont_use_directly flag definitions */

15 -#define POPEN_FLAG_NONE 0

16 -#define POPEN_FLAG_CLOSE_FD (1 << 0) // Close all file descriptors other than STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO

17 -

18 -// the flags to be used by default

19 -#define POPEN_FLAGS_DEFAULT (POPEN_FLAG_CLOSE_FD)

20 -

21 -// mypopen_raw is the interface to use instead of custom_popene_variadic_internal_dont_use_directly()

22 -// mypopen_raw will add the terminating NULL at the arguments list

23 -// we append the parameter 'command' twice - this is because the underlying call needs the command to execute and the argv[0] to pass to it

24 -#define netdata_popen_raw_default_flags_and_environment(pidptr, fpp_child_input, fpp_child_output, command, args...) netdata_popene_variadic_internal_dont_use_directly(pidptr, environ, POPEN_FLAGS_DEFAULT, fpp_child_input, fpp_child_output, command, command, ##args, NULL)

25 -#define netdata_popen_raw_default_flags(pidptr, env, fpp_child_input, fpp_child_output, command, args...) netdata_popene_variadic_internal_dont_use_directly(pidptr, env, POPEN_FLAGS_DEFAULT, fpp_child_input, fpp_child_output, command, command, ##args, NULL)

26 -#define netdata_popen_raw(pidptr, env, flags, fpp_child_input, fpp_child_output, command, args...) netdata_popene_variadic_internal_dont_use_directly(pidptr, env, flags, fpp_child_input, fpp_child_output, command, command, ##args, NULL)

27 -

28 -FILE *netdata_popen(const char *command, volatile pid_t *pidptr, FILE **fp_child_input);

29 -FILE *netdata_popene(const char *command, volatile pid_t *pidptr, char **env, FILE **fp_child_input);

30 -int netdata_popene_variadic_internal_dont_use_directly(volatile pid_t *pidptr, char **env, uint8_t flags, FILE **fpp_child_input, FILE **fpp_child_output, const char *command, ...);

31 -int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid);

32 -

33 -int netdata_spawn(const char *command, volatile pid_t *pidptr);

34 -

35 -#endif /* NETDATA_POPEN_H */

src/libnetdata/procfile/procfile.c

+1 -1

@@ -336,7 +336,7 @@ __attribute__((constructor)) void procfile_initialize_default_separators(void) {

336 if(unlikely(i == '\n' || i == '\r'))

337 procfile_default_separators[i] = PF_CHAR_IS_NEWLINE;

338

339 - else if(unlikely(isspace(i) || !isprint(i)))

339 + else if(unlikely(isspace(i) || (!isprint(i) && !IS_UTF8_BYTE(i))))

340 procfile_default_separators[i] = PF_CHAR_IS_SEPARATOR;

341

342 else

src/libnetdata/socket/socket.c

+4 -6

@@ -194,11 +194,9 @@ int sock_setreuse(int fd, int reuse) {

194 void sock_setcloexec(int fd)

195 {

196 UNUSED(fd);

197 -#ifndef SOCK_CLOEXEC

197 int flags = fcntl(fd, F_GETFD);

198 if (flags != -1)

199 (void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC);

201 -#endif

200 }

201

202 int sock_setreuse_port(int fd __maybe_unused, int reuse __maybe_unused) {

@@ -290,7 +288,7 @@ int create_listen_socket_unix(const char *path, int listen_backlog) {

288 name.sun_family = AF_UNIX;

289 strncpy(name.sun_path, path, sizeof(name.sun_path)-1);

290

293 - errno = 0;

291 + errno_clear();

292 if (unlink(path) == -1 && errno != ENOENT)

293 nd_log(NDLS_DAEMON, NDLP_ERR,

294 "LISTENER: failed to remove existing (probably obsolete or left-over) file on UNIX socket path '%s'.",

@@ -918,7 +916,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t

916 }

917 sock_setcloexec(fd);

918

921 - errno = 0;

919 + errno_clear();

920 if(connect(fd, ai->ai_addr, ai->ai_addrlen) < 0) {

921 if(errno == EALREADY || errno == EINPROGRESS) {

922 nd_log(NDLS_DAEMON, NDLP_DEBUG,

@@ -1200,7 +1198,7 @@ inline int wait_on_socket_or_cancel_with_timeout(

1198 const int wait_ms = (timeout_ms >= ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS || forever) ?

1199 ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS : timeout_ms;

1200

1203 - errno = 0;

1201 + errno_clear();

1202

1203 // check every wait_ms

1204 const int ret = poll(&pfd, 1, wait_ms);

@@ -1482,7 +1480,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien

1480 break;

1481 }

1482 if (!connection_allowed(nfd, client_ip, client_host, hostsize, access_list, "connection", allow_dns)) {

1485 - errno = 0;

1483 + errno_clear();

1484 nd_log(NDLS_DAEMON, NDLP_WARNING,

1485 "Permission denied for client '%s', port '%s'",

1486 client_ip, client_port);

src/libnetdata/spawn_server/spawn_popen.c new

+138

@@ -0,0 +1,138 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "spawn_popen.h"

4 +

5 +SPAWN_SERVER *netdata_main_spawn_server = NULL;

6 +

7 +bool netdata_main_spawn_server_init(const char *name, int argc, const char **argv) {

8 + if(netdata_main_spawn_server == NULL) {

9 + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;

10 + spinlock_lock(&spinlock);

11 + if(netdata_main_spawn_server == NULL)

12 + netdata_main_spawn_server = spawn_server_create(name, NULL, argc, argv);

13 + spinlock_unlock(&spinlock);

14 + }

15 +

16 + return netdata_main_spawn_server != NULL;

17 +}

18 +

19 +void netdata_main_spawn_server_cleanup(void) {

20 + if(netdata_main_spawn_server) {

21 + spawn_server_destroy(netdata_main_spawn_server);

22 + netdata_main_spawn_server = NULL;

23 + }

24 +}

25 +

26 +POPEN_INSTANCE *spawn_popen_run_argv(const char **argv) {

27 + netdata_main_spawn_server_init(NULL, 0, NULL);

28 +

29 + SPAWN_INSTANCE *si = spawn_server_exec(netdata_main_spawn_server, nd_log_collectors_fd(),

30 + 0, argv, NULL, 0, SPAWN_INSTANCE_TYPE_EXEC);

31 +

32 + if(si == NULL) return NULL;

33 +

34 + POPEN_INSTANCE *pi = mallocz(sizeof(*pi));

35 + pi->si = si;

36 + pi->child_stdin_fp = fdopen(spawn_server_instance_write_fd(si), "w");

37 + pi->child_stdout_fp = fdopen(spawn_server_instance_read_fd(si), "r");

38 +

39 + if(!pi->child_stdin_fp) {

40 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot open FILE on child's stdin on fd %d.", spawn_server_instance_write_fd(si));

41 + goto cleanup;

42 + }

43 +

44 + if(!pi->child_stdout_fp) {

45 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot open FILE on child's stdout on fd %d.", spawn_server_instance_read_fd(si));

46 + goto cleanup;

47 + }

48 +

49 + return pi;

50 +

51 +cleanup:

52 + if(pi->child_stdin_fp) { fclose(pi->child_stdin_fp); spawn_server_instance_write_fd(si); }

53 + if(pi->child_stdout_fp) { fclose(pi->child_stdout_fp); spawn_server_instance_read_fd_unset(si); }

54 + spawn_server_exec_kill(netdata_main_spawn_server, si);

55 + freez(pi);

56 + return NULL;

57 +}

58 +

59 +POPEN_INSTANCE *spawn_popen_run_variadic(const char *cmd, ...) {

60 + va_list args;

61 + va_list args_copy;

62 + int argc = 0;

63 +

64 + // Start processing variadic arguments

65 + va_start(args, cmd);

66 +

67 + // Make a copy of args to count the number of arguments

68 + va_copy(args_copy, args);

69 + while (va_arg(args_copy, char *) != NULL) argc++;

70 + va_end(args_copy);

71 +

72 + // Allocate memory for argv array (+2 for cmd and NULL terminator)

73 + const char *argv[argc + 2];

74 +

75 + // Populate the argv array

76 + argv[0] = cmd;

77 +

78 + for (int i = 1; i <= argc; i++)

79 + argv[i] = va_arg(args, const char *);

80 +

81 + argv[argc + 1] = NULL; // NULL-terminate the array

82 +

83 + // End processing variadic arguments

84 + va_end(args);

85 +

86 + return spawn_popen_run_argv(argv);

87 +}

88 +

89 +POPEN_INSTANCE *spawn_popen_run(const char *cmd) {

90 + if(!cmd || !*cmd) return NULL;

91 +

92 + const char *argv[] = {

93 + "/bin/sh",

94 + "-c",

95 + cmd,

96 + NULL

97 + };

98 + return spawn_popen_run_argv(argv);

99 +}

100 +

101 +static int spawn_popen_status_rc(int status) {

102 + if(WIFEXITED(status))

103 + return WEXITSTATUS(status);

104 +

105 + if(WIFSIGNALED(status)) {

106 + int sig = WTERMSIG(status);

107 + switch(sig) {

108 + case SIGTERM:

109 + case SIGPIPE:

110 + return 0;

111 +

112 + default:

113 + return -1;

114 + }

115 + }

116 +

117 + return -1;

118 +}

119 +

120 +int spawn_popen_wait(POPEN_INSTANCE *pi) {

121 + if(!pi) return -1;

122 +

123 + fclose(pi->child_stdin_fp); pi->child_stdin_fp = NULL; spawn_server_instance_write_fd_unset(pi->si);

124 + fclose(pi->child_stdout_fp); pi->child_stdout_fp = NULL; spawn_server_instance_read_fd_unset(pi->si);

125 + int status = spawn_server_exec_wait(netdata_main_spawn_server, pi->si);

126 + freez(pi);

127 + return spawn_popen_status_rc(status);

128 +}

129 +

130 +int spawn_popen_kill(POPEN_INSTANCE *pi) {

131 + if(!pi) return -1;

132 +

133 + fclose(pi->child_stdin_fp); pi->child_stdin_fp = NULL; spawn_server_instance_write_fd_unset(pi->si);

134 + fclose(pi->child_stdout_fp); pi->child_stdout_fp = NULL; spawn_server_instance_read_fd_unset(pi->si);

135 + int status = spawn_server_exec_kill(netdata_main_spawn_server, pi->si);

136 + freez(pi);

137 + return spawn_popen_status_rc(status);

138 +}

src/libnetdata/spawn_server/spawn_popen.h new

+24

@@ -0,0 +1,24 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef SPAWN_POPEN_H

4 +#define SPAWN_POPEN_H

5 +

6 +#include "../libnetdata.h"

7 +

8 +extern SPAWN_SERVER *netdata_main_spawn_server;

9 +bool netdata_main_spawn_server_init(const char *name, int argc, const char **argv);

10 +void netdata_main_spawn_server_cleanup(void);

11 +

12 +typedef struct {

13 + SPAWN_INSTANCE *si;

14 + FILE *child_stdin_fp;

15 + FILE *child_stdout_fp;

16 +} POPEN_INSTANCE;

17 +

18 +POPEN_INSTANCE *spawn_popen_run(const char *cmd);

19 +POPEN_INSTANCE *spawn_popen_run_argv(const char **argv);

20 +POPEN_INSTANCE *spawn_popen_run_variadic(const char *cmd, ...);

21 +int spawn_popen_wait(POPEN_INSTANCE *pi);

22 +int spawn_popen_kill(POPEN_INSTANCE *pi);

23 +

24 +#endif //SPAWN_POPEN_H

src/libnetdata/spawn_server/spawn_server.c new

+1318

@@ -0,0 +1,1318 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "../libnetdata.h"

4 +

5 +#include "spawn_server.h"

6 +

7 +#if defined(OS_WINDOWS)

8 +#include <windows.h>

9 +#include <io.h>

10 +#include <fcntl.h>

11 +#include <process.h>

12 +#include <sys/cygwin.h>

13 +#endif

14 +

15 +struct spawn_server {

16 + size_t id;

17 + size_t request_id;

18 + const char *name;

19 +#if !defined(OS_WINDOWS)

20 + int pipe[2];

21 + int server_sock;

22 + pid_t server_pid;

23 + char *path;

24 + spawn_request_callback_t cb;

25 +

26 + int argc;

27 + const char **argv;

28 + size_t argv0_size;

29 +#endif

30 +};

31 +

32 +struct spawm_instance {

33 + size_t request_id;

34 + int client_sock;

35 + int write_fd;

36 + int read_fd;

37 + pid_t child_pid;

38 +

39 +#if defined(OS_WINDOWS)

40 + HANDLE process_handle;

41 + HANDLE read_handle;

42 + HANDLE write_handle;

43 +#endif

44 +};

45 +

46 +int spawn_server_instance_read_fd(SPAWN_INSTANCE *si) { return si->read_fd; }

47 +int spawn_server_instance_write_fd(SPAWN_INSTANCE *si) { return si->write_fd; }

48 +pid_t spawn_server_instance_pid(SPAWN_INSTANCE *si) { return si->child_pid; }

49 +void spawn_server_instance_read_fd_unset(SPAWN_INSTANCE *si) { si->read_fd = -1; }

50 +void spawn_server_instance_write_fd_unset(SPAWN_INSTANCE *si) { si->write_fd = -1; }

51 +

52 +#if defined(OS_WINDOWS)

53 +

54 +SPAWN_SERVER* spawn_server_create(const char *name, spawn_request_callback_t cb __maybe_unused, int argc __maybe_unused, const char **argv __maybe_unused) {

55 + SPAWN_SERVER* server = callocz(1, sizeof(SPAWN_SERVER));

56 + if(name)

57 + server->name = strdupz(name);

58 + return server;

59 +}

60 +

61 +void spawn_server_destroy(SPAWN_SERVER *server) {

62 + if (server) {

63 + if(server->name) freez((void *)server->name);

64 + freez(server);

65 + }

66 +}

67 +

68 +static BUFFER *argv_to_windows(const char **argv) {

69 + BUFFER *wb = buffer_create(0, NULL);

70 +

71 + // argv[0] is the path

72 + char b[strlen(argv[0]) * 2 + 1024];

73 + cygwin_conv_path(CCP_POSIX_TO_WIN_A | CCP_ABSOLUTE, argv[0], b, sizeof(b));

74 +

75 + buffer_strcat(wb, "cmd.exe /C ");

76 +

77 + for(size_t i = 0; argv[i] ;i++) {

78 + const char *s = (i == 0) ? b : argv[i];

79 + size_t len = strlen(s);

80 + buffer_need_bytes(wb, len * 2 + 1);

81 +

82 + bool needs_quotes = false;

83 + for(const char *c = s; !needs_quotes && *c ; c++) {

84 + switch(*c) {

85 + case ' ':

86 + case '\v':

87 + case '\t':

88 + case '\n':

89 + case '"':

90 + needs_quotes = true;

91 + break;

92 +

93 + default:

94 + break;

95 + }

96 + }

97 +

98 + if(needs_quotes && buffer_strlen(wb))

99 + buffer_strcat(wb, " \"");

100 + else

101 + buffer_putc(wb, ' ');

102 +

103 + for(const char *c = s; *c ; c++) {

104 + switch(*c) {

105 + case '"':

106 + buffer_putc(wb, '\\');

107 + // fall through

108 +

109 + default:

110 + buffer_putc(wb, *c);

111 + break;

112 + }

113 + }

114 +

115 + if(needs_quotes)

116 + buffer_strcat(wb, "\"");

117 + }

118 +

119 + return wb;

120 +}

121 +

122 +SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) {

123 + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;

124 +

125 + if (type != SPAWN_INSTANCE_TYPE_EXEC)

126 + return NULL;

127 +

128 + int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 };

129 +

130 + errno_clear();

131 +

132 + SPAWN_INSTANCE *instance = callocz(1, sizeof(*instance));

133 + instance->request_id = __atomic_add_fetch(&server->request_id, 1, __ATOMIC_RELAXED);

134 +

135 + CLEAN_BUFFER *wb = argv_to_windows(argv);

136 + char *command = (char *)buffer_tostring(wb);

137 +

138 + if (pipe(pipe_stdin) == -1) {

139 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

140 + "SPAWN PARENT: Cannot create stdin pipe() for request No %zu, command: %s",

141 + instance->request_id, command);

142 + goto cleanup;

143 + }

144 +

145 + if (pipe(pipe_stdout) == -1) {

146 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

147 + "SPAWN PARENT: Cannot create stdout pipe() for request No %zu, command: %s",

148 + instance->request_id, command);

149 + goto cleanup;

150 + }

151 +

152 + // do not run multiple times this section

153 + // to prevent handles leaking

154 + spinlock_lock(&spinlock);

155 +

156 + // Convert POSIX file descriptors to Windows handles

157 + HANDLE stdin_read_handle = (HANDLE)_get_osfhandle(pipe_stdin[0]);

158 + HANDLE stdout_write_handle = (HANDLE)_get_osfhandle(pipe_stdout[1]);

159 + HANDLE stderr_handle = (HANDLE)_get_osfhandle(stderr_fd);

160 +

161 + if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_handle == INVALID_HANDLE_VALUE) {

162 + spinlock_unlock(&spinlock);

163 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

164 + "SPAWN PARENT: Invalid handle value(s) for request No %zu, command: %s",

165 + instance->request_id, command);

166 + goto cleanup;

167 + }

168 +

169 + // Set handle inheritance

170 + if (!SetHandleInformation(stdin_read_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||

171 + !SetHandleInformation(stdout_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||

172 + !SetHandleInformation(stderr_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {

173 + spinlock_unlock(&spinlock);

174 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

175 + "SPAWN PARENT: Cannot set handle(s) inheritance for request No %zu, command: %s",

176 + instance->request_id, command);

177 + goto cleanup;

178 + }

179 +

180 + // Set up the STARTUPINFO structure

181 + STARTUPINFO si;

182 + PROCESS_INFORMATION pi;

183 + ZeroMemory(&si, sizeof(si));

184 + si.cb = sizeof(si);

185 + si.dwFlags = STARTF_USESTDHANDLES;

186 + si.hStdInput = stdin_read_handle;

187 + si.hStdOutput = stdout_write_handle;

188 + si.hStdError = stderr_handle;

189 +

190 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

191 + "SPAWN PARENT: Running request No %zu, command: %s",

192 + instance->request_id, command);

193 +

194 + // Spawn the process

195 + if (!CreateProcess(NULL, command, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi)) {

196 + spinlock_unlock(&spinlock);

197 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

198 + "SPAWN PARENT: cannot CreateProcess() for request No %zu, command: %s",

199 + instance->request_id, command);

200 + goto cleanup;

201 + }

202 +

203 + CloseHandle(pi.hThread);

204 +

205 + // end of the critical section

206 + spinlock_unlock(&spinlock);

207 +

208 + // Close unused pipe ends

209 + close(pipe_stdin[0]); pipe_stdin[0] = -1;

210 + close(pipe_stdout[1]); pipe_stdout[1] = -1;

211 +

212 + // Store process information in instance

213 + instance->child_pid = cygwin_winpid_to_pid(pi.dwProcessId);

214 + if(instance->child_pid == -1) instance->child_pid = pi.dwProcessId;

215 +

216 + instance->process_handle = pi.hProcess;

217 +

218 + // Convert handles to POSIX file descriptors

219 + instance->write_fd = pipe_stdin[1];

220 + instance->read_fd = pipe_stdout[0];

221 +

222 + errno_clear();

223 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

224 + "SPAWN PARENT: created process for request No %zu, pid %d, command: %s",

225 + instance->request_id, (int)instance->child_pid, command);

226 +

227 + return instance;

228 +

229 +cleanup:

230 + if (pipe_stdin[0] >= 0) close(pipe_stdin[0]);

231 + if (pipe_stdin[1] >= 0) close(pipe_stdin[1]);

232 + if (pipe_stdout[0] >= 0) close(pipe_stdout[0]);

233 + if (pipe_stdout[1] >= 0) close(pipe_stdout[1]);

234 + freez(instance);

235 + return NULL;

236 +}

237 +

238 +int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *instance) {

239 + if(instance->read_fd != -1) { close(instance->read_fd); instance->read_fd = -1; }

240 + if(instance->write_fd != -1) { close(instance->write_fd); instance->write_fd = -1; }

241 + CloseHandle(instance->read_handle); instance->read_handle = NULL;

242 + CloseHandle(instance->write_handle); instance->write_handle = NULL;

243 +

244 + TerminateProcess(instance->process_handle, 0);

245 +

246 + DWORD exit_code;

247 + GetExitCodeProcess(instance->process_handle, &exit_code);

248 + CloseHandle(instance->process_handle);

249 +

250 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

251 + "SPAWN PARENT: child of request No %zu, pid %d, killed and exited with code %d",

252 + instance->request_id, (int)instance->child_pid, (int)exit_code);

253 +

254 + freez(instance);

255 + return (int)exit_code;

256 +}

257 +

258 +int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *instance) {

259 + if(instance->read_fd != -1) { close(instance->read_fd); instance->read_fd = -1; }

260 + if(instance->write_fd != -1) { close(instance->write_fd); instance->write_fd = -1; }

261 + CloseHandle(instance->read_handle); instance->read_handle = NULL;

262 + CloseHandle(instance->write_handle); instance->write_handle = NULL;

263 +

264 + WaitForSingleObject(instance->process_handle, INFINITE);

265 +

266 + DWORD exit_code = -1;

267 + GetExitCodeProcess(instance->process_handle, &exit_code);

268 + CloseHandle(instance->process_handle);

269 +

270 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

271 + "SPAWN PARENT: child of request No %zu, pid %d, waited and exited with code %d",

272 + instance->request_id, (int)instance->child_pid, (int)exit_code);

273 +

274 + freez(instance);

275 + return (int)exit_code;

276 +}

277 +

278 +#else // !OS_WINDOWS

279 +

280 +#ifdef __APPLE__

281 +#include <crt_externs.h>

282 +#define environ (*_NSGetEnviron())

283 +#else

284 +extern char **environ;

285 +#endif

286 +

287 +static size_t spawn_server_id = 0;

288 +static volatile bool spawn_server_exit = false;

289 +static volatile bool spawn_server_sigchld = false;

290 +static SPAWN_REQUEST *spawn_server_requests = NULL;

291 +

292 +// --------------------------------------------------------------------------------------------------------------------

293 +

294 +static int connect_to_spawn_server(const char *path, bool log) {

295 + int sock = -1;

296 +

297 + if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {

298 + if(log)

299 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: cannot create socket() to connect to spawn server.");

300 + return -1;

301 + }

302 +

303 + struct sockaddr_un server_addr = {

304 + .sun_family = AF_UNIX,

305 + };

306 + strcpy(server_addr.sun_path, path);

307 +

308 + if (connect(sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) == -1) {

309 + if(log)

310 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Cannot connect() to spawn server.");

311 + close(sock);

312 + return -1;

313 + }

314 +

315 + return sock;

316 +}

317 +

318 +// --------------------------------------------------------------------------------------------------------------------

319 +// the child created by the spawn server

320 +

321 +typedef enum __attribute__((packed)) {

322 + STATUS_REPORT_STARTED,

323 + STATUS_REPORT_FAILED,

324 + STATUS_REPORT_EXITED,

325 + STATUS_REPORT_PING,

326 +} STATUS_REPORT;

327 +

328 +struct status_report {

329 + STATUS_REPORT status;

330 + union {

331 + struct {

332 + pid_t pid;

333 + } started;

334 +

335 + struct {

336 + int err_no;

337 + } failed;

338 +

339 + struct {

340 + int waitpid_status;

341 + } exited;

342 + };

343 +};

344 +

345 +static void spawn_server_send_status_ping(int fd) {

346 + struct status_report sr = {

347 + .status = STATUS_REPORT_PING,

348 + };

349 +

350 + if(write(fd, &sr, sizeof(sr)) != sizeof(sr))

351 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Cannot send ping status report");

352 +}

353 +

354 +static void spawn_server_send_status_success(int fd) {

355 + const struct status_report sr = {

356 + .status = STATUS_REPORT_STARTED,

357 + .started = {

358 + .pid = getpid(),

359 + },

360 + };

361 +

362 + if(write(fd, &sr, sizeof(sr)) != sizeof(sr))

363 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Cannot send success status report");

364 +}

365 +

366 +static void spawn_server_send_status_failure(int fd) {

367 + struct status_report sr = {

368 + .status = STATUS_REPORT_FAILED,

369 + .failed = {

370 + .err_no = errno,

371 + },

372 + };

373 +

374 + if(write(fd, &sr, sizeof(sr)) != sizeof(sr))

375 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Cannot send failure status report");

376 +}

377 +

378 +static void spawn_server_send_status_exit(int fd, int waitpid_status) {

379 + struct status_report sr = {

380 + .status = STATUS_REPORT_EXITED,

381 + .exited = {

382 + .waitpid_status = waitpid_status,

383 + },

384 + };

385 +

386 + if(write(fd, &sr, sizeof(sr)) != sizeof(sr))

387 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Cannot send exit status report");

388 +}

389 +

390 +static void spawn_server_run_child(SPAWN_SERVER *server, SPAWN_REQUEST *request) {

391 + // fprintf(stderr, "CHILD: running request %zu on pid %d\n", request->request_id, getpid());

392 +

393 + // close the server sockets;

394 + close(server->server_sock); server->server_sock = -1;

395 + if(server->pipe[0] != -1) { close(server->pipe[0]); server->pipe[0] = -1; }

396 + if(server->pipe[1] != -1) { close(server->pipe[1]); server->pipe[1] = -1; }

397 +

398 + // set the process name

399 + {

400 + char buf[15];

401 + snprintfz(buf, sizeof(buf), "chld-%zu-r%zu", server->id, request->request_id);

402 + os_setproctitle(buf, server->argc, server->argv);

403 + }

404 +

405 + // get the fds from the request

406 + int stdin_fd = request->fds[0];

407 + int stdout_fd = request->fds[1];

408 + int stderr_fd = request->fds[2];

409 + int custom_fd = request->fds[3];

410 +

411 + // change stdio fds to the ones in the request

412 + if (dup2(stdin_fd, STDIN_FILENO) == -1) {

413 + spawn_server_send_status_failure(stdout_fd);

414 + exit(1);

415 + }

416 + if (dup2(stdout_fd, STDOUT_FILENO) == -1) {

417 + spawn_server_send_status_failure(stdout_fd);

418 + exit(1);

419 + }

420 + if (dup2(stderr_fd, STDERR_FILENO) == -1) {

421 + spawn_server_send_status_failure(stdout_fd);

422 + exit(1);

423 + }

424 +

425 + // close the excess fds

426 + close(stdin_fd); stdin_fd = request->fds[0] = STDIN_FILENO;

427 + close(stdout_fd); stdout_fd = request->fds[1] = STDOUT_FILENO;

428 + close(stderr_fd); stderr_fd = request->fds[2] = STDERR_FILENO;

429 +

430 + // overwrite the process environment

431 + environ = (char **)request->environment;

432 +

433 + // Perform different actions based on the type

434 + switch (request->type) {

435 +

436 + case SPAWN_INSTANCE_TYPE_EXEC:

437 + spawn_server_send_status_success(request->socket);

438 + close(request->socket); request->socket = -1;

439 + close(custom_fd); custom_fd = -1;

440 + execvp(request->argv[0], (char **)request->argv);

441 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

442 + "SPAWN SERVER: Failed to execute command of request No %zu (argv[0] = '%s')",

443 + request->request_id, request->argv[0]);

444 + exit(1);

445 + break;

446 +

447 + case SPAWN_INSTANCE_TYPE_CALLBACK:

448 + if(server->cb == NULL) {

449 + errno = ENOENT;

450 + spawn_server_send_status_failure(request->socket);

451 + close(request->socket); request->socket = -1;

452 + exit(1);

453 + }

454 + spawn_server_send_status_success(request->socket);

455 + close(request->socket); request->socket = -1;

456 + server->cb(request);

457 + exit(0);

458 + break;

459 +

460 + default:

461 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: unknown request type %u", request->type);

462 + exit(1);

463 + }

464 +}

465 +

466 +// --------------------------------------------------------------------------------------------------------------------

467 +// Encoding and decoding of spawn server request argv type of data

468 +

469 +// Function to encode argv or envp

470 +static void* encode_argv(const char **argv, size_t *out_size) {

471 + size_t buffer_size = 1024; // Initial buffer size

472 + size_t buffer_used = 0;

473 + char *buffer = mallocz(buffer_size);

474 +

475 + if(argv) {

476 + for (const char **p = argv; *p != NULL; p++) {

477 + if (strlen(*p) == 0)

478 + continue; // Skip empty strings

479 +

480 + size_t len = strlen(*p) + 1;

481 + size_t wanted_size = buffer_used + len + 1;

482 +

483 + if (wanted_size >= buffer_size) {

484 + buffer_size *= 2;

485 +

486 + if(buffer_size < wanted_size)

487 + buffer_size = wanted_size;

488 +

489 + buffer = reallocz(buffer, buffer_size);

490 + }

491 +

492 + memcpy(&buffer[buffer_used], *p, len);

493 + buffer_used += len;

494 + }

495 + }

496 +

497 + buffer[buffer_used++] = '\0'; // Final empty string

498 + *out_size = buffer_used;

499 +

500 + return buffer;

501 +}

502 +

503 +// Function to decode argv or envp

504 +static const char** decode_argv(const char *buffer, size_t size) {

505 + size_t count = 0;

506 + const char *ptr = buffer;

507 + while (ptr < buffer + size) {

508 + if(ptr && *ptr) {

509 + count++;

510 + ptr += strlen(ptr) + 1;

511 + }

512 + else

513 + break;

514 + }

515 +

516 + const char **argv = mallocz((count + 1) * sizeof(char *));

517 +

518 + ptr = buffer;

519 + for (size_t i = 0; i < count; i++) {

520 + argv[i] = ptr;

521 + ptr += strlen(ptr) + 1;

522 + }

523 + argv[count] = NULL; // Null-terminate the array

524 +

525 + return argv;

526 +}

527 +

528 +// --------------------------------------------------------------------------------------------------------------------

529 +// Sending and receiving requests

530 +

531 +typedef enum __attribute__((packed)) {

532 + SPAWN_SERVER_MSG_INVALID = 0,

533 + SPAWN_SERVER_MSG_REQUEST,

534 + SPAWN_SERVER_MSG_PING,

535 +} SPAWN_SERVER_MSG;

536 +

537 +static bool spawn_server_is_running(const char *path) {

538 + struct msghdr msg = {0};

539 + struct iovec iov[6];

540 + SPAWN_SERVER_MSG msg_type = SPAWN_SERVER_MSG_PING;

541 + size_t dummy_size = 0;

542 + SPAWN_INSTANCE_TYPE dummy_type = 0;

543 + char cmsgbuf[CMSG_SPACE(sizeof(int))];

544 +

545 + iov[0].iov_base = &msg_type;

546 + iov[0].iov_len = sizeof(msg_type);

547 +

548 + iov[1].iov_base = &dummy_size;

549 + iov[1].iov_len = sizeof(dummy_size);

550 +

551 + iov[2].iov_base = &dummy_size;

552 + iov[2].iov_len = sizeof(dummy_size);

553 +

554 + iov[3].iov_base = &dummy_size;

555 + iov[3].iov_len = sizeof(dummy_size);

556 +

557 + iov[4].iov_base = &dummy_size;

558 + iov[4].iov_len = sizeof(dummy_size);

559 +

560 + iov[5].iov_base = &dummy_type;

561 + iov[5].iov_len = sizeof(dummy_type);

562 +

563 + msg.msg_iov = iov;

564 + msg.msg_iovlen = 6;

565 + msg.msg_control = cmsgbuf;

566 + msg.msg_controllen = sizeof(cmsgbuf);

567 +

568 + int sock = connect_to_spawn_server(path, false);

569 + if(sock == -1)

570 + return false;

571 +

572 + int rc = sendmsg(sock, &msg, 0);

573 + if (rc < 0) {

574 + // cannot send the message

575 + close(sock);

576 + return false;

577 + }

578 +

579 + // Receive response

580 + struct status_report sr = { 0 };

581 + if (read(sock, &sr, sizeof(sr)) != sizeof(sr)) {

582 + // cannot receive a ping reply

583 + close(sock);

584 + return false;

585 + }

586 +

587 + close(sock);

588 + return sr.status == STATUS_REPORT_PING;

589 +}

590 +

591 +static bool spawn_server_send_request(SPAWN_REQUEST *request) {

592 + bool ret = false;

593 +

594 + size_t env_size = 0;

595 + void *encoded_env = encode_argv(request->environment, &env_size);

596 + if (!encoded_env)

597 + goto cleanup;

598 +

599 + size_t argv_size = 0;

600 + void *encoded_argv = encode_argv(request->argv, &argv_size);

601 + if (!encoded_argv)

602 + goto cleanup;

603 +

604 + struct msghdr msg = {0};

605 + struct cmsghdr *cmsg;

606 + SPAWN_SERVER_MSG msg_type = SPAWN_SERVER_MSG_REQUEST;

607 + char cmsgbuf[CMSG_SPACE(sizeof(int) * SPAWN_SERVER_TRANSFER_FDS)];

608 + struct iovec iov[10];

609 +

610 +

611 + // We send 1 request with 10 iovec in it

612 + // The request will be received in 2 parts

613 + // 1. the first 6 iovec which include the sizes of the memory allocations required

614 + // 2. the last 4 iovec which require the memory allocations to be received

615 +

616 + iov[0].iov_base = &msg_type;

617 + iov[0].iov_len = sizeof(msg_type);

618 +

619 + iov[1].iov_base = &request->request_id;

620 + iov[1].iov_len = sizeof(request->request_id);

621 +

622 + iov[2].iov_base = &env_size;

623 + iov[2].iov_len = sizeof(env_size);

624 +

625 + iov[3].iov_base = &argv_size;

626 + iov[3].iov_len = sizeof(argv_size);

627 +

628 + iov[4].iov_base = &request->data_size;

629 + iov[4].iov_len = sizeof(request->data_size);

630 +

631 + iov[5].iov_base = &request->type; // Added this line

632 + iov[5].iov_len = sizeof(request->type);

633 +

634 + iov[6].iov_base = encoded_env;

635 + iov[6].iov_len = env_size;

636 +

637 + iov[7].iov_base = encoded_argv;

638 + iov[7].iov_len = argv_size;

639 +

640 + iov[8].iov_base = (char *)request->data;

641 + iov[8].iov_len = request->data_size;

642 +

643 + iov[9].iov_base = NULL;

644 + iov[9].iov_len = 0;

645 +

646 + msg.msg_iov = iov;

647 + msg.msg_iovlen = 10;

648 + msg.msg_control = cmsgbuf;

649 + msg.msg_controllen = CMSG_SPACE(sizeof(int) * SPAWN_SERVER_TRANSFER_FDS);

650 +

651 + cmsg = CMSG_FIRSTHDR(&msg);

652 + cmsg->cmsg_level = SOL_SOCKET;

653 + cmsg->cmsg_type = SCM_RIGHTS;

654 + cmsg->cmsg_len = CMSG_LEN(sizeof(int) * SPAWN_SERVER_TRANSFER_FDS);

655 +

656 + memcpy(CMSG_DATA(cmsg), request->fds, sizeof(int) * SPAWN_SERVER_TRANSFER_FDS);

657 +

658 + int rc = sendmsg(request->socket, &msg, 0);

659 +

660 + if (rc < 0) {

661 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Failed to sendmsg() request to spawn server using socket %d.", request->socket);

662 + goto cleanup;

663 + }

664 + else {

665 + ret = true;

666 + // fprintf(stderr, "PARENT: sent request %zu on socket %d (fds: %d, %d, %d, %d) from tid %d\n",

667 + // request->request_id, request->socket, request->fds[0], request->fds[1], request->fds[2], request->fds[3], os_gettid());

668 + }

669 +

670 +cleanup:

671 + freez(encoded_env);

672 + freez(encoded_argv);

673 + return ret;

674 +}

675 +

676 +static void spawn_server_receive_request(int sock, SPAWN_SERVER *server) {

677 + struct msghdr msg = {0};

678 + struct iovec iov[6];

679 + SPAWN_SERVER_MSG msg_type = SPAWN_SERVER_MSG_INVALID;

680 + size_t request_id;

681 + size_t env_size;

682 + size_t argv_size;

683 + size_t data_size;

684 + SPAWN_INSTANCE_TYPE type;

685 + char cmsgbuf[CMSG_SPACE(sizeof(int) * SPAWN_SERVER_TRANSFER_FDS)];

686 + char *envp = NULL, *argv = NULL, *data = NULL;

687 + int stdin_fd = -1, stdout_fd = -1, stderr_fd = -1, custom_fd = -1;

688 +

689 + // First recvmsg() to read sizes and control message

690 + iov[0].iov_base = &msg_type;

691 + iov[0].iov_len = sizeof(msg_type);

692 + iov[1].iov_base = &request_id;

693 + iov[1].iov_len = sizeof(request_id);

694 + iov[2].iov_base = &env_size;

695 + iov[2].iov_len = sizeof(env_size);

696 + iov[3].iov_base = &argv_size;

697 + iov[3].iov_len = sizeof(argv_size);

698 + iov[4].iov_base = &data_size;

699 + iov[4].iov_len = sizeof(data_size);

700 + iov[5].iov_base = &type;

701 + iov[5].iov_len = sizeof(type);

702 +

703 + msg.msg_iov = iov;

704 + msg.msg_iovlen = 6;

705 + msg.msg_control = cmsgbuf;

706 + msg.msg_controllen = sizeof(cmsgbuf);

707 +

708 + if (recvmsg(sock, &msg, 0) < 0) {

709 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: failed to recvmsg() the first part of the request.");

710 + return;

711 + }

712 +

713 + if(msg_type == SPAWN_SERVER_MSG_PING) {

714 + spawn_server_send_status_ping(sock);

715 + return;

716 + }

717 +

718 + // Extract file descriptors from control message

719 + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);

720 + if (cmsg == NULL || cmsg->cmsg_len != CMSG_LEN(sizeof(int) * SPAWN_SERVER_TRANSFER_FDS)) {

721 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

722 + "SPAWN SERVER: Received invalid control message (expected %zu bytes, received %zu bytes)",

723 + CMSG_LEN(sizeof(int) * SPAWN_SERVER_TRANSFER_FDS), cmsg?cmsg->cmsg_len:0);

724 + return;

725 + }

726 +

727 + if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {

728 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Received unexpected control message type.");

729 + return;

730 + }

731 +

732 + int *fds = (int *)CMSG_DATA(cmsg);

733 + stdin_fd = fds[0];

734 + stdout_fd = fds[1];

735 + stderr_fd = fds[2];

736 + custom_fd = fds[3];

737 +

738 + if (stdin_fd < 0 || stdout_fd < 0 || stderr_fd < 0) {

739 + nd_log(NDLS_COLLECTORS, NDLP_ERR,

740 + "SPAWN SERVER: invalid file descriptors received, stdin = %d, stdout = %d, stderr = %d",

741 + stdin_fd, stdout_fd, stderr_fd);

742 + goto cleanup;

743 + }

744 +

745 + // Second recvmsg() to read buffer contents

746 + iov[0].iov_base = envp = mallocz(env_size);

747 + iov[0].iov_len = env_size;

748 + iov[1].iov_base = argv = mallocz(argv_size);

749 + iov[1].iov_len = argv_size;

750 + iov[2].iov_base = data = mallocz(data_size);

751 + iov[2].iov_len = data_size;

752 +

753 + msg.msg_iov = iov;

754 + msg.msg_iovlen = 3;

755 + msg.msg_control = NULL;

756 + msg.msg_controllen = 0;

757 +

758 + ssize_t total_bytes_received = recvmsg(sock, &msg, 0);

759 + if (total_bytes_received < 0) {

760 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: failed to recvmsg() the second part of the request.");

761 + goto cleanup;

762 + }

763 +

764 + // fprintf(stderr, "SPAWN SERVER: received request %zu (fds: %d, %d, %d, %d)\n", request_id,

765 + // stdin_fd, stdout_fd, stderr_fd, custom_fd);

766 +

767 + SPAWN_REQUEST *request = mallocz(sizeof(*request));

768 + *request = (SPAWN_REQUEST){

769 + .pid = 0,

770 + .request_id = request_id,

771 + .socket = sock,

772 + .fds = {

773 + [0] = stdin_fd,

774 + [1] = stdout_fd,

775 + [2] = stderr_fd,

776 + [3] = custom_fd,

777 + },

778 + .environment = decode_argv(envp, env_size),

779 + .argv = decode_argv(argv, argv_size),

780 + .data = data,

781 + .data_size = data_size,

782 + .type = type

783 + };

784 +

785 + pid_t pid = fork();

786 + if (pid == 0) {

787 + // the child

788 + spawn_server_run_child(server, request);

789 + exit(1);

790 +

791 + }

792 + else if (pid > 0) {

793 + // the parent

794 + request->pid = pid;

795 + request->environment = NULL;

796 + request->argv = NULL;

797 + request->data = NULL;

798 + request->data_size = 0;

799 + request->fds[0] = -1;

800 + request->fds[1] = -1;

801 + request->fds[2] = -1;

802 + request->fds[3] = -1;

803 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(spawn_server_requests, request, prev, next);

804 +

805 + // do not fork this socket on other children

806 + sock_setcloexec(request->socket);

807 + }

808 + else {

809 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to fork() child.");

810 + spawn_server_send_status_failure(stdout_fd);

811 + freez(request);

812 + }

813 +

814 +cleanup:

815 + if(stdin_fd != -1) close(stdin_fd);

816 + if(stdout_fd != -1) close(stdout_fd);

817 + if(stderr_fd != -1) close(stderr_fd);

818 + if(custom_fd != -1) close(custom_fd);

819 + freez(envp);

820 + freez(argv);

821 + freez(data);

822 +}

823 +

824 +// --------------------------------------------------------------------------------------------------------------------

825 +// the spawn server main event loop

826 +

827 +static void spawn_server_sigchld_handler(int signo __maybe_unused) {

828 + spawn_server_sigchld = true;

829 +}

830 +

831 +static void spawn_server_sigterm_handler(int signo __maybe_unused) {

832 + spawn_server_exit = true;

833 +}

834 +

835 +static SPAWN_REQUEST *find_request_by_pid(pid_t pid) {

836 + for(SPAWN_REQUEST *rq = spawn_server_requests; rq ;rq = rq->next)

837 + if(rq->pid == pid)

838 + return rq;

839 +

840 + return NULL;

841 +}

842 +

843 +static void spawn_server_process_sigchld(void) {

844 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: checking for exited children");

845 +

846 + int status;

847 + pid_t pid;

848 +

849 + // Loop to check for exited child processes

850 + while ((pid = waitpid((pid_t)(-1), &status, WNOHANG)) != 0) {

851 + if(pid == -1)

852 + break;

853 +

854 + SPAWN_REQUEST *rq = find_request_by_pid(pid);

855 + size_t request_id = rq ? rq->request_id : 0;

856 + bool send_report_remove_request = false;

857 +

858 + if(WIFEXITED(status)) {

859 + nd_log(NDLS_COLLECTORS, NDLP_INFO,

860 + "SPAWN SERVER: child with pid %d (request %zu) exited normally with exit code %d",

861 + pid, request_id, WEXITSTATUS(status));

862 + send_report_remove_request = true;

863 + }

864 + else if(WIFSIGNALED(status)) {

865 + if(WCOREDUMP(status))

866 + nd_log(NDLS_COLLECTORS, NDLP_INFO,

867 + "SPAWN SERVER: child with pid %d (request %zu) coredump'd due to signal %d",

868 + pid, request_id, WTERMSIG(status));

869 + else

870 + nd_log(NDLS_COLLECTORS, NDLP_INFO,

871 + "SPAWN SERVER: child with pid %d (request %zu) killed by signal %d",

872 + pid, request_id, WTERMSIG(status));

873 + send_report_remove_request = true;

874 + }

875 + else if(WIFSTOPPED(status)) {

876 + nd_log(NDLS_COLLECTORS, NDLP_INFO,

877 + "SPAWN SERVER: child with pid %d (request %zu) stopped due to signal %d",

878 + pid, request_id, WSTOPSIG(status));

879 + send_report_remove_request = false;

880 + }

881 + else if(WIFCONTINUED(status)) {

882 + nd_log(NDLS_COLLECTORS, NDLP_INFO,

883 + "SPAWN SERVER: child with pid %d (request %zu) continued due to signal %d",

884 + pid, request_id, SIGCONT);

885 + send_report_remove_request = false;

886 + }

887 + else {

888 + nd_log(NDLS_COLLECTORS, NDLP_INFO,

889 + "SPAWN SERVER: child with pid %d (request %zu) reports unhandled status",

890 + pid, request_id);

891 + send_report_remove_request = false;

892 + }

893 +

894 + if(send_report_remove_request && rq) {

895 + spawn_server_send_status_exit(rq->socket, status);

896 + close(rq->socket);

897 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(spawn_server_requests, rq, prev, next);

898 + freez(rq);

899 + }

900 + }

901 +}

902 +

903 +static void signals_unblock(void) {

904 + sigset_t sigset;

905 + sigfillset(&sigset);

906 +

907 + if(pthread_sigmask(SIG_UNBLOCK, &sigset, NULL) == -1) {

908 + netdata_log_error("SIGNAL: Could not unblock signals for threads");

909 + }

910 +}

911 +

912 +static void spawn_server_event_loop(SPAWN_SERVER *server) {

913 + int pipe_fd = server->pipe[1];

914 + close(server->pipe[0]); server->pipe[0] = -1;

915 +

916 + signals_unblock();

917 +

918 + // Set up the signal handler for SIGCHLD and SIGTERM

919 + struct sigaction sa;

920 + sa.sa_handler = spawn_server_sigchld_handler;

921 + sigemptyset(&sa.sa_mask);

922 + sa.sa_flags = SA_RESTART | SA_NOCLDSTOP;

923 + if (sigaction(SIGCHLD, &sa, NULL) == -1) {

924 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: sigaction() failed for SIGCHLD");

925 + exit(1);

926 + }

927 +

928 + sa.sa_handler = spawn_server_sigterm_handler;

929 + if (sigaction(SIGTERM, &sa, NULL) == -1) {

930 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: sigaction() failed for SIGTERM");

931 + exit(1);

932 + }

933 +

934 + struct status_report sr = {

935 + .status = STATUS_REPORT_STARTED,

936 + .started = {

937 + .pid = getpid(),

938 + },

939 + };

940 + if (write(pipe_fd, &sr, sizeof(sr)) != sizeof(sr)) {

941 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: failed to write initial status report.");

942 + exit(1);

943 + }

944 +

945 + struct pollfd fds[2];

946 + fds[0].fd = server->server_sock;

947 + fds[0].events = POLLIN;

948 + fds[1].fd = pipe_fd;

949 + fds[1].events = POLLHUP | POLLERR;

950 +

951 + while(!spawn_server_exit) {

952 + int ret = poll(fds, 2, -1);

953 + if (spawn_server_sigchld) {

954 + spawn_server_sigchld = false;

955 + spawn_server_process_sigchld();

956 +

957 + if(ret == -1)

958 + continue;

959 + }

960 +

961 + if (ret == -1) {

962 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: poll() failed");

963 + break;

964 + }

965 +

966 + if (fds[1].revents & (POLLHUP|POLLERR)) {

967 + // Pipe has been closed (parent has exited)

968 + nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "SPAWN SERVER: Parent process has exited");

969 + break;

970 + }

971 +

972 + if (fds[0].revents & POLLIN) {

973 + int client_sock = accept(server->server_sock, NULL, NULL);

974 + if (client_sock == -1) {

975 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: accept() failed");

976 + continue;

977 + }

978 +

979 + spawn_server_receive_request(client_sock, server);

980 + }

981 + }

982 +

983 + // Cleanup before exiting

984 + unlink(server->path);

985 +

986 + // stop all children

987 + if(spawn_server_requests) {

988 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: killing all children...");

989 + size_t killed = 0;

990 + for(SPAWN_REQUEST *rq = spawn_server_requests; rq ; rq = rq->next) {

991 + kill(rq->pid, SIGTERM);

992 + killed++;

993 + }

994 + while(spawn_server_requests) {

995 + spawn_server_process_sigchld();

996 + tinysleep();

997 + }

998 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: all %zu children finished", killed);

999 + }

1000 +

1001 + exit(1);

1002 +}

1003 +

1004 +// --------------------------------------------------------------------------------------------------------------------

1005 +// management of the spawn server

1006 +

1007 +void spawn_server_destroy(SPAWN_SERVER *server) {

1008 + if(server->pipe[0] != -1) close(server->pipe[0]);

1009 + if(server->pipe[1] != -1) close(server->pipe[1]);

1010 + if(server->server_sock != -1) close(server->server_sock);

1011 +

1012 + if(server->server_pid) {

1013 + kill(server->server_pid, SIGTERM);

1014 + waitpid(server->server_pid, NULL, 0);

1015 + }

1016 +

1017 + if(server->path) {

1018 + unlink(server->path);

1019 + freez(server->path);

1020 + }

1021 +

1022 + freez((void *)server->name);

1023 + freez(server);

1024 +}

1025 +

1026 +static bool spawn_server_create_listening_socket(SPAWN_SERVER *server) {

1027 + if(spawn_server_is_running(server->path)) {

1028 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Server is already listening on path '%s'", server->path);

1029 + return false;

1030 + }

1031 +

1032 + if ((server->server_sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {

1033 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to create socket()");

1034 + return false;

1035 + }

1036 +

1037 + struct sockaddr_un server_addr = {

1038 + .sun_family = AF_UNIX,

1039 + };

1040 + strcpy(server_addr.sun_path, server->path);

1041 + unlink(server->path);

1042 + errno = 0;

1043 +

1044 + if (bind(server->server_sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) == -1) {

1045 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to bind()");

1046 + return false;

1047 + }

1048 +

1049 + if (listen(server->server_sock, 5) == -1) {

1050 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to listen()");

1051 + return false;

1052 + }

1053 +

1054 + return true;

1055 +}

1056 +

1057 +static void replace_stdio_with_dev_null() {

1058 + int dev_null_fd = open("/dev/null", O_RDWR);

1059 + if (dev_null_fd == -1) {

1060 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno));

1061 + return;

1062 + }

1063 +

1064 + // Redirect stdin (fd 0)

1065 + if (dup2(dev_null_fd, STDIN_FILENO) == -1) {

1066 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno));

1067 + close(dev_null_fd);

1068 + return;

1069 + }

1070 +

1071 + // Redirect stdout (fd 1)

1072 + if (dup2(dev_null_fd, STDOUT_FILENO) == -1) {

1073 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno));

1074 + close(dev_null_fd);

1075 + return;

1076 + }

1077 +

1078 + // Close the original /dev/null file descriptor

1079 + close(dev_null_fd);

1080 +}

1081 +

1082 +SPAWN_SERVER* spawn_server_create(const char *name, spawn_request_callback_t child_callback, int argc, const char **argv) {

1083 + SPAWN_SERVER *server = callocz(1, sizeof(SPAWN_SERVER));

1084 + server->pipe[0] = -1;

1085 + server->pipe[1] = -1;

1086 + server->server_sock = -1;

1087 + server->cb = child_callback;

1088 + server->argc = argc;

1089 + server->argv = argv;

1090 + server->argv0_size = (argv && argv[0]) ? strlen(argv[0]) : 0;

1091 +

1092 + server->id = __atomic_add_fetch(&spawn_server_id, 1, __ATOMIC_RELAXED);

1093 +

1094 + char *runtime_directory = getenv("NETDATA_CACHE_DIR");

1095 + if(runtime_directory && !*runtime_directory) runtime_directory = NULL;

1096 + if (runtime_directory) {

1097 + struct stat statbuf;

1098 +

1099 + if(!*runtime_directory)

1100 + // it is empty

1101 + runtime_directory = NULL;

1102 +

1103 + else if (stat(runtime_directory, &statbuf) == 0 && S_ISDIR(statbuf.st_mode)) {

1104 + // it exists and it is a directory

1105 +

1106 + if (access(runtime_directory, W_OK) != 0) {

1107 + // it is not writable by us

1108 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Runtime directory '%s' is not writable, falling back to '/tmp'", runtime_directory);

1109 + runtime_directory = NULL;

1110 + }

1111 + }

1112 + else {

1113 + // it does not exist

1114 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Runtime directory '%s' does not exist, falling back to '/tmp'", runtime_directory);

1115 + runtime_directory = NULL;

1116 + }

1117 + }

1118 + if(!runtime_directory)

1119 + runtime_directory = "/tmp";

1120 +

1121 + char path[1024];

1122 + if(name && *name) {

1123 + server->name = strdupz(name);

1124 + snprintf(path, sizeof(path), "%s/.netdata-spawn-%s.sock", runtime_directory, name);

1125 + }

1126 + else {

1127 + snprintfz(path, sizeof(path), "%d-%zu", getpid(), server->id);

1128 + server->name = strdupz(path);

1129 + snprintf(path, sizeof(path), "%s/.netdata-spawn-%d-%zu.sock", runtime_directory, getpid(), server->id);

1130 + }

1131 +

1132 + server->path = strdupz(path);

1133 +

1134 + if (!spawn_server_create_listening_socket(server))

1135 + goto cleanup;

1136 +

1137 + if (pipe(server->pipe) == -1) {

1138 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Cannot create status pipe()");

1139 + goto cleanup;

1140 + }

1141 +

1142 + pid_t pid = fork();

1143 + if (pid == 0) {

1144 + // the child - the spawn server

1145 + {

1146 + char buf[15];

1147 + snprintfz(buf, sizeof(buf), "spawn-%s", server->name);

1148 + os_setproctitle(buf, server->argc, server->argv);

1149 + }

1150 +

1151 + replace_stdio_with_dev_null();

1152 + os_close_all_non_std_open_fds_except((int[]){ server->server_sock, server->pipe[1] }, 2);

1153 + spawn_server_event_loop(server);

1154 + }

1155 + else if (pid > 0) {

1156 + // the parent

1157 + server->server_pid = pid;

1158 + close(server->server_sock); server->server_sock = -1;

1159 + close(server->pipe[1]); server->pipe[1] = -1;

1160 +

1161 + struct status_report sr = { 0 };

1162 + if (read(server->pipe[0], &sr, sizeof(sr)) != sizeof(sr)) {

1163 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: cannot read() initial status report from spawn server");

1164 + goto cleanup;

1165 + }

1166 +

1167 + if(sr.status != STATUS_REPORT_STARTED) {

1168 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: server did not respond with success.");

1169 + goto cleanup;

1170 + }

1171 +

1172 + if(sr.started.pid != server->server_pid) {

1173 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: server sent pid %d but we have created %d.", sr.started.pid, server->server_pid);

1174 + goto cleanup;

1175 + }

1176 +

1177 + return server;

1178 + }

1179 +

1180 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Cannot fork()");

1181 +

1182 +cleanup:

1183 + spawn_server_destroy(server);

1184 + return NULL;

1185 +}

1186 +

1187 +// --------------------------------------------------------------------------------------------------------------------

1188 +// creating spawn server instances

1189 +

1190 +void spawn_server_exec_destroy(SPAWN_INSTANCE *instance) {

1191 + if(instance->child_pid) kill(instance->child_pid, SIGTERM);

1192 + if(instance->write_fd != -1) close(instance->write_fd);

1193 + if(instance->read_fd != -1) close(instance->read_fd);

1194 + if(instance->client_sock != -1) close(instance->client_sock);

1195 + freez(instance);

1196 +}

1197 +

1198 +int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *instance) {

1199 + int rc = -1;

1200 +

1201 + // close the child pipes, to make it exit

1202 + if(instance->write_fd != -1) { close(instance->write_fd); instance->write_fd = -1; }

1203 + if(instance->read_fd != -1) { close(instance->read_fd); instance->read_fd = -1; }

1204 +

1205 + // get the result

1206 + struct status_report sr = { 0 };

1207 + if(read(instance->client_sock, &sr, sizeof(sr)) != sizeof(sr))

1208 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: failed to receive final status report for child %d, request %zu", instance->child_pid, instance->request_id);

1209 +

1210 + else switch(sr.status) {

1211 + case STATUS_REPORT_EXITED:

1212 + rc = sr.exited.waitpid_status;

1213 + break;

1214 +

1215 + case STATUS_REPORT_STARTED:

1216 + case STATUS_REPORT_FAILED:

1217 + default:

1218 + errno = 0;

1219 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: invalid status report to exec spawn request %zu for pid %d (status = %u)", instance->request_id, instance->child_pid, sr.status);

1220 + break;

1221 + }

1222 +

1223 + instance->child_pid = 0;

1224 + spawn_server_exec_destroy(instance);

1225 + return rc;

1226 +}

1227 +

1228 +int spawn_server_exec_kill(SPAWN_SERVER *server, SPAWN_INSTANCE *instance) {

1229 + // kill the child, if it is still running

1230 + if(instance->child_pid) kill(instance->child_pid, SIGTERM);

1231 + return spawn_server_exec_wait(server, instance);

1232 +}

1233 +

1234 +SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd, const char **argv, const void *data, size_t data_size, SPAWN_INSTANCE_TYPE type) {

1235 + int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 };

1236 +

1237 + SPAWN_INSTANCE *instance = callocz(1, sizeof(SPAWN_INSTANCE));

1238 + instance->read_fd = -1;

1239 + instance->write_fd = -1;

1240 +

1241 + instance->client_sock = connect_to_spawn_server(server->path, true);

1242 + if(instance->client_sock == -1)

1243 + goto cleanup;

1244 +

1245 + if (pipe(pipe_stdin) == -1) {

1246 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Cannot create stdin pipe()");

1247 + goto cleanup;

1248 + }

1249 +

1250 + if (pipe(pipe_stdout) == -1) {

1251 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Cannot create stdout pipe()");

1252 + goto cleanup;

1253 + }

1254 +

1255 + SPAWN_REQUEST request = {

1256 + .request_id = __atomic_add_fetch(&server->request_id, 1, __ATOMIC_RELAXED),

1257 + .socket = instance->client_sock,

1258 + .fds = {

1259 + [0] = pipe_stdin[0],

1260 + [1] = pipe_stdout[1],

1261 + [2] = stderr_fd,

1262 + [3] = custom_fd,

1263 + },

1264 + .environment = (const char **)environ,

1265 + .argv = argv,

1266 + .data = data,

1267 + .data_size = data_size,

1268 + .type = type

1269 + };

1270 +

1271 + if(!spawn_server_send_request(&request))

1272 + goto cleanup;

1273 +

1274 + close(pipe_stdin[0]); pipe_stdin[0] = -1;

1275 + instance->write_fd = pipe_stdin[1]; pipe_stdin[1] = -1;

1276 +

1277 + close(pipe_stdout[1]); pipe_stdout[1] = -1;

1278 + instance->read_fd = pipe_stdout[0]; pipe_stdout[0] = -1;

1279 +

1280 + struct status_report sr = { 0 };

1281 + if(read(instance->client_sock, &sr, sizeof(sr)) != sizeof(sr)) {

1282 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Failed to exec spawn request %zu (cannot get initial status report)", request.request_id);

1283 + goto cleanup;

1284 + }

1285 +

1286 + switch(sr.status) {

1287 + case STATUS_REPORT_STARTED:

1288 + instance->child_pid = sr.started.pid;

1289 + return instance;

1290 +

1291 + case STATUS_REPORT_FAILED:

1292 + errno = sr.failed.err_no;

1293 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Failed to exec spawn request %zu (check errno #1)", request.request_id);

1294 + errno = 0;

1295 + break;

1296 +

1297 + case STATUS_REPORT_EXITED:

1298 + errno = ENOEXEC;

1299 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Failed to exec spawn request %zu (check errno #2)", request.request_id);

1300 + errno = 0;

1301 + break;

1302 +

1303 + default:

1304 + errno = 0;

1305 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Invalid status report to exec spawn request %zu (received invalid data)", request.request_id);

1306 + break;

1307 + }

1308 +

1309 +cleanup:

1310 + if (pipe_stdin[0] >= 0) close(pipe_stdin[0]);

1311 + if (pipe_stdin[1] >= 0) close(pipe_stdin[1]);

1312 + if (pipe_stdout[0] >= 0) close(pipe_stdout[0]);

1313 + if (pipe_stdout[1] >= 0) close(pipe_stdout[1]);

1314 + spawn_server_exec_destroy(instance);

1315 + return NULL;

1316 +}

1317 +

1318 +#endif // !OS_WINDOWS

src/libnetdata/spawn_server/spawn_server.h new

+48

@@ -0,0 +1,48 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef SPAWN_SERVER_H

4 +#define SPAWN_SERVER_H

5 +

6 +#define SPAWN_SERVER_TRANSFER_FDS 4

7 +

8 +typedef enum {

9 + SPAWN_INSTANCE_TYPE_EXEC = 0,

10 +#if !defined(OS_WINDOWS)

11 + SPAWN_INSTANCE_TYPE_CALLBACK = 1

12 +#endif

13 +} SPAWN_INSTANCE_TYPE;

14 +

15 +// this is only used publicly for SPAWN_INSTANCE_TYPE_CALLBACK

16 +// which is not available in Windows

17 +typedef struct spawn_request {

18 + size_t request_id;

19 + pid_t pid;

20 + int socket;

21 + int fds[SPAWN_SERVER_TRANSFER_FDS]; // 0 = stdin, 1 = stdout, 2 = stderr, 3 = custom

22 + const char **environment;

23 + const char **argv;

24 + const void *data;

25 + size_t data_size;

26 + SPAWN_INSTANCE_TYPE type;

27 + struct spawn_request *prev, *next;

28 +} SPAWN_REQUEST;

29 +

30 +typedef void (*spawn_request_callback_t)(SPAWN_REQUEST *request);

31 +

32 +typedef struct spawm_instance SPAWN_INSTANCE;

33 +typedef struct spawn_server SPAWN_SERVER;

34 +

35 +SPAWN_SERVER* spawn_server_create(const char *name, spawn_request_callback_t child_callback, int argc, const char **argv);

36 +void spawn_server_destroy(SPAWN_SERVER *server);

37 +

38 +SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd, const char **argv, const void *data, size_t data_size, SPAWN_INSTANCE_TYPE type);

39 +int spawn_server_exec_kill(SPAWN_SERVER *server, SPAWN_INSTANCE *instance);

40 +int spawn_server_exec_wait(SPAWN_SERVER *server, SPAWN_INSTANCE *instance);

41 +

42 +int spawn_server_instance_read_fd(SPAWN_INSTANCE *si);

43 +int spawn_server_instance_write_fd(SPAWN_INSTANCE *si);

44 +pid_t spawn_server_instance_pid(SPAWN_INSTANCE *si);

45 +void spawn_server_instance_read_fd_unset(SPAWN_INSTANCE *si);

46 +void spawn_server_instance_write_fd_unset(SPAWN_INSTANCE *si);

47 +

48 +#endif //SPAWN_SERVER_H

src/logsmanagement/helper.h

+2 -2

@@ -95,7 +95,7 @@ static inline str2xx_errno str2int(int *out, char *s, int base) {

95 // m_assert(0, "str2int error: STR2XX_INCONVERTIBLE");

96 return STR2XX_INCONVERTIBLE;

97 }

98 - errno = 0;

98 + errno_clear();

99 long l = strtol(s, &end, base);

100 /* Both checks are needed because INT_MAX == LONG_MAX is possible. */

101 if (unlikely(l > INT_MAX || (errno == ERANGE && l == LONG_MAX))){

@@ -124,7 +124,7 @@ static inline str2xx_errno str2float(float *out, char *s) {

124 // m_assert(0, "str2float error: STR2XX_INCONVERTIBLE");

125 return STR2XX_INCONVERTIBLE;

126 }

127 - errno = 0;

127 + errno_clear();

128 float f = strtof(s, &end);

129 /* Both checks are needed because INT_MAX == LONG_MAX is possible. */

130 if (unlikely((errno == ERANGE && f == HUGE_VALF))){

src/registry/registry_db.c

+1 -1

@@ -162,7 +162,7 @@ int registry_db_save(void) {

162

163 fclose(fp);

164

165 - errno = 0;

165 + errno_clear();

166

167 // remove the .old db

168 netdata_log_debug(D_REGISTRY, "REGISTRY: Removing old db '%s'", old_filename);

src/spawn/README.md

src/spawn/spawn.c deleted

-288

@@ -1,288 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include "spawn.h"

4 -

5 -static uv_thread_t thread;

6 -int spawn_thread_error;

7 -int spawn_thread_shutdown;

8 -

9 -struct spawn_queue spawn_cmd_queue;

10 -

11 -static struct spawn_cmd_info *create_spawn_cmd(const char *command_to_run)

12 -{

13 - struct spawn_cmd_info *cmdinfo;

14 -

15 - cmdinfo = mallocz(sizeof(*cmdinfo));

16 - fatal_assert(0 == uv_cond_init(&cmdinfo->cond));

17 - fatal_assert(0 == uv_mutex_init(&cmdinfo->mutex));

18 - cmdinfo->serial = 0; /* invalid */

19 - cmdinfo->command_to_run = strdupz(command_to_run);

20 - cmdinfo->exit_status = -1; /* invalid */

21 - cmdinfo->pid = -1; /* invalid */

22 - cmdinfo->flags = 0;

23 -

24 - return cmdinfo;

25 -}

26 -

27 -void destroy_spawn_cmd(struct spawn_cmd_info *cmdinfo)

28 -{

29 - uv_cond_destroy(&cmdinfo->cond);

30 - uv_mutex_destroy(&cmdinfo->mutex);

31 -

32 - freez(cmdinfo->command_to_run);

33 - freez(cmdinfo);

34 -}

35 -

36 -int spawn_cmd_compare(void *a, void *b)

37 -{

38 - struct spawn_cmd_info *cmda = a, *cmdb = b;

39 -

40 - /* No need for mutex, serial will never change and the entries cannot be deallocated yet */

41 - if (cmda->serial < cmdb->serial) return -1;

42 - if (cmda->serial > cmdb->serial) return 1;

43 -

44 - return 0;

45 -}

46 -

47 -static void init_spawn_cmd_queue(void)

48 -{

49 - spawn_cmd_queue.cmd_tree.root = NULL;

50 - spawn_cmd_queue.cmd_tree.compar = spawn_cmd_compare;

51 - spawn_cmd_queue.size = 0;

52 - spawn_cmd_queue.latest_serial = 0;

53 - fatal_assert(0 == uv_cond_init(&spawn_cmd_queue.cond));

54 - fatal_assert(0 == uv_mutex_init(&spawn_cmd_queue.mutex));

55 -}

56 -

57 -/*

58 - * Returns serial number of the enqueued command

59 - */

60 -uint64_t spawn_enq_cmd(const char *command_to_run)

61 -{

62 - unsigned queue_size;

63 - uint64_t serial;

64 - avl_t *avl_ret;

65 - struct spawn_cmd_info *cmdinfo;

66 -

67 - cmdinfo = create_spawn_cmd(command_to_run);

68 -

69 - /* wait for free space in queue */

70 - uv_mutex_lock(&spawn_cmd_queue.mutex);

71 - while ((queue_size = spawn_cmd_queue.size) == SPAWN_MAX_OUTSTANDING) {

72 - uv_cond_wait(&spawn_cmd_queue.cond, &spawn_cmd_queue.mutex);

73 - }

74 - fatal_assert(queue_size < SPAWN_MAX_OUTSTANDING);

75 - spawn_cmd_queue.size = queue_size + 1;

76 -

77 - serial = ++spawn_cmd_queue.latest_serial; /* 0 is invalid */

78 - cmdinfo->serial = serial; /* No need to take the cmd mutex since it is unreachable at the moment */

79 -

80 - /* enqueue command */

81 - avl_ret = avl_insert(&spawn_cmd_queue.cmd_tree, (avl_t *)cmdinfo);

82 - fatal_assert(avl_ret == (avl_t *)cmdinfo);

83 - uv_mutex_unlock(&spawn_cmd_queue.mutex);

84 -

85 - /* wake up event loop */

86 - fatal_assert(0 == uv_async_send(&spawn_async));

87 - return serial;

88 -}

89 -

90 -/*

91 - * Blocks until command with serial finishes running. Only one thread is allowed to wait per command.

92 - */

93 -void spawn_wait_cmd(uint64_t serial, int *exit_status, time_t *exec_run_timestamp)

94 -{

95 - avl_t *avl_ret;

96 - struct spawn_cmd_info tmp, *cmdinfo;

97 -

98 - tmp.serial = serial;

99 -

100 - uv_mutex_lock(&spawn_cmd_queue.mutex);

101 - avl_ret = avl_search(&spawn_cmd_queue.cmd_tree, (avl_t *)&tmp);

102 - uv_mutex_unlock(&spawn_cmd_queue.mutex);

103 -

104 - fatal_assert(avl_ret); /* Could be NULL if more than 1 threads wait for the command */

105 - cmdinfo = (struct spawn_cmd_info *)avl_ret;

106 -

107 - uv_mutex_lock(&cmdinfo->mutex);

108 - while (!(cmdinfo->flags & SPAWN_CMD_DONE)) {

109 - /* Only 1 thread is allowed to wait for this command to finish */

110 - uv_cond_wait(&cmdinfo->cond, &cmdinfo->mutex);

111 - }

112 - uv_mutex_unlock(&cmdinfo->mutex);

113 -

114 - spawn_deq_cmd(cmdinfo);

115 - *exit_status = cmdinfo->exit_status;

116 - *exec_run_timestamp = cmdinfo->exec_run_timestamp;

117 -

118 - destroy_spawn_cmd(cmdinfo);

119 -}

120 -

121 -void spawn_deq_cmd(struct spawn_cmd_info *cmdinfo)

122 -{

123 - unsigned queue_size;

124 - avl_t *avl_ret;

125 -

126 - uv_mutex_lock(&spawn_cmd_queue.mutex);

127 - queue_size = spawn_cmd_queue.size;

128 - fatal_assert(queue_size);

129 - /* dequeue command */

130 - avl_ret = avl_remove(&spawn_cmd_queue.cmd_tree, (avl_t *)cmdinfo);

131 - fatal_assert(avl_ret);

132 -

133 - spawn_cmd_queue.size = queue_size - 1;

134 -

135 - /* wake up callers */

136 - uv_cond_signal(&spawn_cmd_queue.cond);

137 - uv_mutex_unlock(&spawn_cmd_queue.mutex);

138 -}

139 -

140 -/*

141 - * Must be called from the spawn client event loop context. This way no mutex is needed because the event loop is the

142 - * only writer as far as struct spawn_cmd_info entries are concerned.

143 - */

144 -static int find_unprocessed_spawn_cmd_cb(void *entry, void *data)

145 -{

146 - struct spawn_cmd_info **cmdinfop = data, *cmdinfo = entry;

147 -

148 - if (!(cmdinfo->flags & SPAWN_CMD_PROCESSED)) {

149 - *cmdinfop = cmdinfo;

150 - return -1; /* break tree traversal */

151 - }

152 - return 0; /* continue traversing */

153 -}

154 -

155 -struct spawn_cmd_info *spawn_get_unprocessed_cmd(void)

156 -{

157 - struct spawn_cmd_info *cmdinfo;

158 - unsigned queue_size;

159 - int ret;

160 -

161 - uv_mutex_lock(&spawn_cmd_queue.mutex);

162 - queue_size = spawn_cmd_queue.size;

163 - if (queue_size == 0) {

164 - uv_mutex_unlock(&spawn_cmd_queue.mutex);

165 - return NULL;

166 - }

167 - /* find command */

168 - cmdinfo = NULL;

169 - ret = avl_traverse(&spawn_cmd_queue.cmd_tree, find_unprocessed_spawn_cmd_cb, (void *)&cmdinfo);

170 - if (-1 != ret) { /* no commands available for processing */

171 - uv_mutex_unlock(&spawn_cmd_queue.mutex);

172 - return NULL;

173 - }

174 - uv_mutex_unlock(&spawn_cmd_queue.mutex);

175 -

176 - return cmdinfo;

177 -}

178 -

179 -/**

180 - * This function spawns a process that shares a libuv IPC pipe with the caller and performs spawn server duties.

181 - * The spawn server process will close all open file descriptors except for the pipe, UV_STDOUT_FD, and UV_STDERR_FD.

182 - * The caller has to be the netdata user as configured.

183 - *

184 - * @param loop the libuv loop of the caller context

185 - * @param spawn_channel the bidirectional libuv IPC pipe that the server and the caller will share

186 - * @param process the spawn server libuv process context

187 - * @return 0 on success or the libuv error code

188 - */

189 -int create_spawn_server(uv_loop_t *loop, uv_pipe_t *spawn_channel, uv_process_t *process)

190 -{

191 - uv_process_options_t options = {0};

192 - char *args[3];

193 - int ret;

194 -#define SPAWN_SERVER_DESCRIPTORS (3)

195 - uv_stdio_container_t stdio[SPAWN_SERVER_DESCRIPTORS];

196 - struct passwd *passwd = NULL;

197 - char *user = NULL;

198 -

199 - passwd = getpwuid(getuid());

200 - user = (passwd && passwd->pw_name) ? passwd->pw_name : "";

201 -

202 - args[0] = netdata_exe_file;

203 - args[1] = SPAWN_SERVER_COMMAND_LINE_ARGUMENT;

204 - args[2] = NULL;

205 -

206 - memset(&options, 0, sizeof(options));

207 - options.file = netdata_exe_file;

208 - options.args = args;

209 - options.exit_cb = NULL; //exit_cb;

210 - options.stdio = stdio;

211 - options.stdio_count = SPAWN_SERVER_DESCRIPTORS;

212 -

213 - stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE | UV_WRITABLE_PIPE;

214 - stdio[0].data.stream = (uv_stream_t *)spawn_channel; /* bidirectional libuv pipe */

215 - stdio[1].flags = UV_INHERIT_FD;

216 - stdio[1].data.fd = 1 /* UV_STDOUT_FD */;

217 - stdio[2].flags = UV_INHERIT_FD;

218 - stdio[2].data.fd = nd_log_health_fd() /* UV_STDERR_FD */;

219 -

220 - ret = uv_spawn(loop, process, &options); /* execute the netdata binary again as the netdata user */

221 - if (0 != ret) {

222 - netdata_log_error("uv_spawn (process: \"%s\") (user: %s) failed (%s).", netdata_exe_file, user, uv_strerror(ret));

223 - fatal("Cannot start netdata without the spawn server.");

224 - }

225 -

226 - return ret;

227 -}

228 -

229 -#define CONCURRENT_SPAWNS 16

230 -#define SPAWN_ITERATIONS 10000

231 -#undef CONCURRENT_STRESS_TEST

232 -

233 -void spawn_init(void)

234 -{

235 - struct completion completion;

236 - int error;

237 -

238 - netdata_log_info("Initializing spawn client.");

239 -

240 - init_spawn_cmd_queue();

241 -

242 - completion_init(&completion);

243 - error = uv_thread_create(&thread, spawn_client, &completion);

244 - if (error) {

245 - netdata_log_error("uv_thread_create(): %s", uv_strerror(error));

246 - goto after_error;

247 - }

248 - /* wait for spawn client thread to initialize */

249 - completion_wait_for(&completion);

250 - completion_destroy(&completion);

251 -

252 - if (spawn_thread_error) {

253 - error = uv_thread_join(&thread);

254 - if (error) {

255 - netdata_log_error("uv_thread_create(): %s", uv_strerror(error));

256 - }

257 - goto after_error;

258 - }

259 -#ifdef CONCURRENT_STRESS_TEST

260 - signals_reset();

261 - signals_unblock();

262 -

263 - sleep(60);

264 - uint64_t serial[CONCURRENT_SPAWNS];

265 - for (int j = 0 ; j < SPAWN_ITERATIONS ; ++j) {

266 - for (int i = 0; i < CONCURRENT_SPAWNS; ++i) {

267 - char cmd[64];

268 - sprintf(cmd, "echo CONCURRENT_STRESS_TEST %d 1>&2", j * CONCURRENT_SPAWNS + i + 1);

269 - serial[i] = spawn_enq_cmd(cmd);

270 - netdata_log_info("Queued command %s for spawning.", cmd);

271 - }

272 - int exit_status;

273 - time_t exec_run_timestamp;

274 - for (int i = 0; i < CONCURRENT_SPAWNS; ++i) {

275 - netdata_log_info("Started waiting for serial %llu exit status %d run timestamp %llu.", serial[i], exit_status,

276 - exec_run_timestamp);

277 - spawn_wait_cmd(serial[i], &exit_status, &exec_run_timestamp);

278 - netdata_log_info("Finished waiting for serial %llu exit status %d run timestamp %llu.", serial[i], exit_status,

279 - exec_run_timestamp);

280 - }

281 - }

282 - exit(0);

283 -#endif

284 - return;

285 -

286 - after_error:

287 - netdata_log_error("Failed to initialize spawn service. The alarms notifications will not be spawned.");

288 -}

src/spawn/spawn.h deleted

-109

@@ -1,109 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_SPAWN_H

4 -#define NETDATA_SPAWN_H 1

5 -

6 -#include "daemon/common.h"

7 -

8 -#define SPAWN_SERVER_COMMAND_LINE_ARGUMENT "--special-spawn-server"

9 -

10 -typedef enum spawn_protocol {

11 - SPAWN_PROT_EXEC_CMD = 0,

12 - SPAWN_PROT_SPAWN_RESULT,

13 - SPAWN_PROT_CMD_EXIT_STATUS

14 -} spawn_prot_t;

15 -

16 -struct spawn_prot_exec_cmd {

17 - uint16_t command_length;

18 - char command_to_run[];

19 -};

20 -

21 -struct spawn_prot_spawn_result {

22 - pid_t exec_pid; /* 0 if failed to spawn */

23 - time_t exec_run_timestamp; /* time of successfully spawning the command */

24 -};

25 -

26 -struct spawn_prot_cmd_exit_status {

27 - int exec_exit_status;

28 -};

29 -

30 -struct spawn_prot_header {

31 - spawn_prot_t opcode;

32 - void *handle;

33 -};

34 -

35 -#undef SPAWN_DEBUG /* define to enable debug prints */

36 -

37 -#define SPAWN_MAX_OUTSTANDING (32768)

38 -

39 -#define SPAWN_CMD_PROCESSED 0x00000001

40 -#define SPAWN_CMD_IN_PROGRESS 0x00000002

41 -#define SPAWN_CMD_FAILED_TO_SPAWN 0x00000004

42 -#define SPAWN_CMD_DONE 0x00000008

43 -

44 -struct spawn_cmd_info {

45 - avl_t avl;

46 -

47 - /* concurrency control per command */

48 - uv_mutex_t mutex;

49 - uv_cond_t cond; /* users block here until command has finished */

50 -

51 - uint64_t serial;

52 - char *command_to_run;

53 - int exit_status;

54 - pid_t pid;

55 - unsigned long flags;

56 - time_t exec_run_timestamp; /* time of successfully spawning the command */

57 -};

58 -

59 -/* spawn command queue */

60 -struct spawn_queue {

61 - avl_tree_type cmd_tree;

62 -

63 - /* concurrency control of command queue */

64 - uv_mutex_t mutex;

65 - uv_cond_t cond;

66 -

67 - volatile unsigned size;

68 - uint64_t latest_serial;

69 -};

70 -

71 -struct write_context {

72 - uv_write_t write_req;

73 - struct spawn_prot_header header;

74 - struct spawn_prot_cmd_exit_status exit_status;

75 - struct spawn_prot_spawn_result spawn_result;

76 - struct spawn_prot_exec_cmd payload;

77 -};

78 -

79 -extern int spawn_thread_error;

80 -extern int spawn_thread_shutdown;

81 -extern uv_async_t spawn_async;

82 -

83 -void spawn_init(void);

84 -void spawn_server(void);

85 -void spawn_client(void *arg);

86 -void destroy_spawn_cmd(struct spawn_cmd_info *cmdinfo);

87 -uint64_t spawn_enq_cmd(const char *command_to_run);

88 -void spawn_wait_cmd(uint64_t serial, int *exit_status, time_t *exec_run_timestamp);

89 -void spawn_deq_cmd(struct spawn_cmd_info *cmdinfo);

90 -struct spawn_cmd_info *spawn_get_unprocessed_cmd(void);

91 -int create_spawn_server(uv_loop_t *loop, uv_pipe_t *spawn_channel, uv_process_t *process);

92 -

93 -/*

94 - * Copies from the source buffer to the protocol buffer. It advances the source buffer by the amount copied. It

95 - * subtracts the amount copied from the source length.

96 - */

97 -static inline void copy_to_prot_buffer(char *prot_buffer, unsigned *prot_buffer_len, unsigned max_to_copy,

98 - char **source, unsigned *source_len)

99 -{

100 - unsigned to_copy;

101 -

102 - to_copy = MIN(max_to_copy, *source_len);

103 - memcpy(prot_buffer + *prot_buffer_len, *source, to_copy);

104 - *prot_buffer_len += to_copy;

105 - *source += to_copy;

106 - *source_len -= to_copy;

107 -}

108 -

109 -#endif //NETDATA_SPAWN_H

src/spawn/spawn_client.c deleted

-250

@@ -1,250 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include "spawn.h"

4 -

5 -static uv_process_t process;

6 -static uv_pipe_t spawn_channel;

7 -static uv_loop_t *loop;

8 -uv_async_t spawn_async;

9 -

10 -static char prot_buffer[MAX_COMMAND_LENGTH];

11 -static unsigned prot_buffer_len = 0;

12 -

13 -static void async_cb(uv_async_t *handle)

14 -{

15 - uv_stop(handle->loop);

16 -}

17 -

18 -static void after_pipe_write(uv_write_t* req, int status)

19 -{

20 - (void)status;

21 -#ifdef SPAWN_DEBUG

22 - netdata_log_info("CLIENT %s called status=%d", __func__, status);

23 -#endif

24 - void **data = req->data;

25 - freez(data[0]);

26 - freez(data[1]);

27 - freez(data);

28 -}

29 -

30 -static void client_parse_spawn_protocol(unsigned source_len, char *source)

31 -{

32 - unsigned required_len;

33 - struct spawn_prot_header *header;

34 - struct spawn_prot_spawn_result *spawn_result;

35 - struct spawn_prot_cmd_exit_status *exit_status;

36 - struct spawn_cmd_info *cmdinfo;

37 -

38 - while (source_len) {

39 - required_len = sizeof(*header);

40 - if (prot_buffer_len < required_len)

41 - copy_to_prot_buffer(prot_buffer, &prot_buffer_len, required_len - prot_buffer_len, &source, &source_len);

42 - if (prot_buffer_len < required_len)

43 - return; /* Source buffer ran out */

44 -

45 - header = (struct spawn_prot_header *)prot_buffer;

46 - cmdinfo = (struct spawn_cmd_info *)header->handle;

47 - fatal_assert(NULL != cmdinfo);

48 -

49 - switch(header->opcode) {

50 - case SPAWN_PROT_SPAWN_RESULT:

51 - required_len += sizeof(*spawn_result);

52 - if (prot_buffer_len < required_len)

53 - copy_to_prot_buffer(prot_buffer, &prot_buffer_len, required_len - prot_buffer_len, &source, &source_len);

54 - if (prot_buffer_len < required_len)

55 - return; /* Source buffer ran out */

56 -

57 - spawn_result = (struct spawn_prot_spawn_result *)(header + 1);

58 - uv_mutex_lock(&cmdinfo->mutex);

59 - cmdinfo->pid = spawn_result->exec_pid;

60 - if (0 == cmdinfo->pid) { /* Failed to spawn */

61 -#ifdef SPAWN_DEBUG

62 - netdata_log_info("CLIENT %s SPAWN_PROT_SPAWN_RESULT failed to spawn.", __func__);

63 -#endif

64 - cmdinfo->flags |= SPAWN_CMD_FAILED_TO_SPAWN | SPAWN_CMD_DONE;

65 - uv_cond_signal(&cmdinfo->cond);

66 - } else {

67 - cmdinfo->exec_run_timestamp = spawn_result->exec_run_timestamp;

68 - cmdinfo->flags |= SPAWN_CMD_IN_PROGRESS;

69 -#ifdef SPAWN_DEBUG

70 - netdata_log_info("CLIENT %s SPAWN_PROT_SPAWN_RESULT in progress.", __func__);

71 -#endif

72 - }

73 - uv_mutex_unlock(&cmdinfo->mutex);

74 - prot_buffer_len = 0;

75 - break;

76 - case SPAWN_PROT_CMD_EXIT_STATUS:

77 - required_len += sizeof(*exit_status);

78 - if (prot_buffer_len < required_len)

79 - copy_to_prot_buffer(prot_buffer, &prot_buffer_len, required_len - prot_buffer_len, &source, &source_len);

80 - if (prot_buffer_len < required_len)

81 - return; /* Source buffer ran out */

82 -

83 - exit_status = (struct spawn_prot_cmd_exit_status *)(header + 1);

84 - uv_mutex_lock(&cmdinfo->mutex);

85 - cmdinfo->exit_status = exit_status->exec_exit_status;

86 -#ifdef SPAWN_DEBUG

87 - netdata_log_info("CLIENT %s SPAWN_PROT_CMD_EXIT_STATUS %d.", __func__, exit_status->exec_exit_status);

88 -#endif

89 - cmdinfo->flags |= SPAWN_CMD_DONE;

90 - uv_cond_signal(&cmdinfo->cond);

91 - uv_mutex_unlock(&cmdinfo->mutex);

92 - prot_buffer_len = 0;

93 - break;

94 - default:

95 - fatal_assert(0);

96 - break;

97 - }

98 -

99 - }

100 -}

101 -

102 -static void on_pipe_read(uv_stream_t* pipe, ssize_t nread, const uv_buf_t* buf)

103 -{

104 - if (0 == nread) {

105 - netdata_log_info("%s: Zero bytes read from spawn pipe.", __func__);

106 - } else if (UV_EOF == nread) {

107 - netdata_log_info("EOF found in spawn pipe.");

108 - } else if (nread < 0) {

109 - netdata_log_error("%s: %s", __func__, uv_strerror(nread));

110 - }

111 -

112 - if (nread < 0) { /* stop stream due to EOF or error */

113 - (void)uv_read_stop((uv_stream_t *)pipe);

114 - } else if (nread) {

115 -#ifdef SPAWN_DEBUG

116 - netdata_log_info("CLIENT %s read %u", __func__, (unsigned)nread);

117 -#endif

118 - client_parse_spawn_protocol(nread, buf->base);

119 - }

120 - if (buf && buf->len) {

121 - freez(buf->base);

122 - }

123 -

124 - if (nread < 0) {

125 - uv_close((uv_handle_t *)pipe, NULL);

126 - }

127 -}

128 -

129 -static void on_read_alloc(uv_handle_t* handle,

130 - size_t suggested_size,

131 - uv_buf_t* buf)

132 -{

133 - (void)handle;

134 - buf->base = mallocz(suggested_size);

135 - buf->len = suggested_size;

136 -}

137 -

138 -static void spawn_process_cmd(struct spawn_cmd_info *cmdinfo)

139 -{

140 - int ret;

141 - uv_buf_t *writebuf;

142 - struct write_context *write_ctx;

143 -

144 - void **data = callocz(2, sizeof(void *));

145 - writebuf = callocz(3, sizeof(uv_buf_t));

146 - write_ctx = callocz(1, sizeof(*write_ctx));

147 -

148 - data[0] = write_ctx;

149 - data[1] = writebuf;

150 - write_ctx->write_req.data = data;

151 -

152 - uv_mutex_lock(&cmdinfo->mutex);

153 - cmdinfo->flags |= SPAWN_CMD_PROCESSED;

154 - uv_mutex_unlock(&cmdinfo->mutex);

155 -

156 - write_ctx->header.opcode = SPAWN_PROT_EXEC_CMD;

157 - write_ctx->header.handle = cmdinfo;

158 - write_ctx->payload.command_length = strlen(cmdinfo->command_to_run);

159 -

160 - writebuf[0] = uv_buf_init((char *)&write_ctx->header, sizeof(write_ctx->header));

161 - writebuf[1] = uv_buf_init((char *)&write_ctx->payload, sizeof(write_ctx->payload));

162 - writebuf[2] = uv_buf_init((char *)cmdinfo->command_to_run, write_ctx->payload.command_length);

163 -

164 -#ifdef SPAWN_DEBUG

165 - netdata_log_info("CLIENT %s SPAWN_PROT_EXEC_CMD %u", __func__, (unsigned)cmdinfo->serial);

166 -#endif

167 - ret = uv_write(&write_ctx->write_req, (uv_stream_t *)&spawn_channel, writebuf, 3, after_pipe_write);

168 - fatal_assert(ret == 0);

169 -}

170 -

171 -void spawn_client(void *arg)

172 -{

173 - uv_thread_set_name_np("DAEMON_SPAWN");

174 -

175 - int ret;

176 - struct completion *completion = (struct completion *)arg;

177 -

178 - loop = mallocz(sizeof(uv_loop_t));

179 - ret = uv_loop_init(loop);

180 - if (ret) {

181 - netdata_log_error("uv_loop_init(): %s", uv_strerror(ret));

182 - spawn_thread_error = ret;

183 - goto error_after_loop_init;

184 - }

185 - loop->data = NULL;

186 -

187 - spawn_async.data = NULL;

188 - ret = uv_async_init(loop, &spawn_async, async_cb);

189 - if (ret) {

190 - netdata_log_error("uv_async_init(): %s", uv_strerror(ret));

191 - spawn_thread_error = ret;

192 - goto error_after_async_init;

193 - }

194 -

195 - ret = uv_pipe_init(loop, &spawn_channel, 1);

196 - if (ret) {

197 - netdata_log_error("uv_pipe_init(): %s", uv_strerror(ret));

198 - spawn_thread_error = ret;

199 - goto error_after_pipe_init;

200 - }

201 - fatal_assert(spawn_channel.ipc);

202 -

203 - ret = create_spawn_server(loop, &spawn_channel, &process);

204 - if (ret) {

205 - netdata_log_error("Failed to fork spawn server process.");

206 - spawn_thread_error = ret;

207 - goto error_after_spawn_server;

208 - }

209 -

210 - spawn_thread_error = 0;

211 - spawn_thread_shutdown = 0;

212 - /* wake up initialization thread */

213 - completion_mark_complete(completion);

214 -

215 - prot_buffer_len = 0;

216 - ret = uv_read_start((uv_stream_t *)&spawn_channel, on_read_alloc, on_pipe_read);

217 - fatal_assert(ret == 0);

218 -

219 - while (spawn_thread_shutdown == 0) {

220 - struct spawn_cmd_info *cmdinfo;

221 -

222 - uv_run(loop, UV_RUN_DEFAULT);

223 - while (NULL != (cmdinfo = spawn_get_unprocessed_cmd())) {

224 - spawn_process_cmd(cmdinfo);

225 - }

226 - }

227 - /* cleanup operations of the event loop */

228 - netdata_log_info("Shutting down spawn client event loop.");

229 - uv_close((uv_handle_t *)&spawn_channel, NULL);

230 - uv_close((uv_handle_t *)&spawn_async, NULL);

231 - uv_run(loop, UV_RUN_DEFAULT); /* flush all libuv handles */

232 -

233 - netdata_log_info("Shutting down spawn client loop complete.");

234 - fatal_assert(0 == uv_loop_close(loop));

235 -

236 - return;

237 -

238 -error_after_spawn_server:

239 - uv_close((uv_handle_t *)&spawn_channel, NULL);

240 -error_after_pipe_init:

241 - uv_close((uv_handle_t *)&spawn_async, NULL);

242 -error_after_async_init:

243 - uv_run(loop, UV_RUN_DEFAULT); /* flush all libuv handles */

244 - fatal_assert(0 == uv_loop_close(loop));

245 -error_after_loop_init:

246 - freez(loop);

247 -

248 - /* wake up initialization thread */

249 - completion_mark_complete(completion);

250 -}

src/spawn/spawn_server.c deleted

-386

@@ -1,386 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include "spawn.h"

4 -

5 -static uv_loop_t *loop;

6 -static uv_pipe_t server_pipe;

7 -

8 -static int server_shutdown = 0;

9 -

10 -static uv_thread_t thread;

11 -

12 -/* spawn outstanding execution structure */

13 -static avl_tree_lock spawn_outstanding_exec_tree;

14 -

15 -static char prot_buffer[MAX_COMMAND_LENGTH];

16 -static unsigned prot_buffer_len = 0;

17 -

18 -struct spawn_execution_info {

19 - avl_t avl;

20 -

21 - void *handle;

22 - int exit_status;

23 - pid_t pid;

24 - struct spawn_execution_info *next;

25 -};

26 -

27 -int spawn_exec_compare(void *a, void *b)

28 -{

29 - struct spawn_execution_info *spwna = a, *spwnb = b;

30 -

31 - if (spwna->pid < spwnb->pid) return -1;

32 - if (spwna->pid > spwnb->pid) return 1;

33 -

34 - return 0;

35 -}

36 -

37 -/* wake up waiter thread to reap the spawned processes */

38 -static uv_mutex_t wait_children_mutex;

39 -static uv_cond_t wait_children_cond;

40 -static uint8_t spawned_processes;

41 -static struct spawn_execution_info *child_waited_list;

42 -static uv_async_t child_waited_async;

43 -

44 -static inline struct spawn_execution_info *dequeue_child_waited_list(void)

45 -{

46 - struct spawn_execution_info *exec_info;

47 -

48 - uv_mutex_lock(&wait_children_mutex);

49 - if (NULL == child_waited_list) {

50 - exec_info = NULL;

51 - } else {

52 - exec_info = child_waited_list;

53 - child_waited_list = exec_info->next;

54 - }

55 - uv_mutex_unlock(&wait_children_mutex);

56 -

57 - return exec_info;

58 -}

59 -

60 -static inline void enqueue_child_waited_list(struct spawn_execution_info *exec_info)

61 -{

62 - uv_mutex_lock(&wait_children_mutex);

63 - exec_info->next = child_waited_list;

64 - child_waited_list = exec_info;

65 - uv_mutex_unlock(&wait_children_mutex);

66 -}

67 -

68 -static void after_pipe_write(uv_write_t *req, int status)

69 -{

70 - (void)status;

71 -#ifdef SPAWN_DEBUG

72 - fprintf(stderr, "SERVER %s called status=%d\n", __func__, status);

73 -#endif

74 - void **data = req->data;

75 - freez(data[0]);

76 - freez(data[1]);

77 - freez(data);

78 -}

79 -

80 -static void child_waited_async_cb(uv_async_t *async_handle)

81 -{

82 - uv_buf_t *writebuf;

83 - int ret;

84 - struct spawn_execution_info *exec_info;

85 - struct write_context *write_ctx;

86 -

87 - (void)async_handle;

88 - while (NULL != (exec_info = dequeue_child_waited_list())) {

89 - write_ctx = mallocz(sizeof(*write_ctx));

90 -

91 - void **data = callocz(2, sizeof(void *));

92 - writebuf = callocz(2, sizeof(uv_buf_t));

93 -

94 - data[0] = write_ctx;

95 - data[1] = writebuf;

96 - write_ctx->write_req.data = data;

97 -

98 - write_ctx->header.opcode = SPAWN_PROT_CMD_EXIT_STATUS;

99 - write_ctx->header.handle = exec_info->handle;

100 - write_ctx->exit_status.exec_exit_status = exec_info->exit_status;

101 - writebuf[0] = uv_buf_init((char *) &write_ctx->header, sizeof(write_ctx->header));

102 - writebuf[1] = uv_buf_init((char *) &write_ctx->exit_status, sizeof(write_ctx->exit_status));

103 -#ifdef SPAWN_DEBUG

104 - fprintf(stderr, "SERVER %s SPAWN_PROT_CMD_EXIT_STATUS\n", __func__);

105 -#endif

106 - ret = uv_write(&write_ctx->write_req, (uv_stream_t *) &server_pipe, writebuf, 2, after_pipe_write);

107 - fatal_assert(ret == 0);

108 -

109 - freez(exec_info);

110 - }

111 -}

112 -

113 -static void wait_children(void *arg)

114 -{

115 - siginfo_t i;

116 - struct spawn_execution_info tmp, *exec_info;

117 - avl_t *ret_avl;

118 -

119 - (void)arg;

120 - while (!server_shutdown) {

121 - uv_mutex_lock(&wait_children_mutex);

122 - while (!spawned_processes) {

123 - uv_cond_wait(&wait_children_cond, &wait_children_mutex);

124 - }

125 - spawned_processes = 0;

126 - uv_mutex_unlock(&wait_children_mutex);

127 -

128 - while (!server_shutdown) {

129 - i.si_pid = 0;

130 - if (os_waitid(P_ALL, (id_t) 0, &i, WEXITED) == -1) {

131 - if (errno != ECHILD)

132 - fprintf(stderr, "SPAWN: Failed to wait: %s\n", strerror(errno));

133 - break;

134 - }

135 - if (i.si_pid == 0) {

136 - fprintf(stderr, "SPAWN: No child exited.\n");

137 - break;

138 - }

139 -#ifdef SPAWN_DEBUG

140 - fprintf(stderr, "SPAWN: Successfully waited for pid:%d.\n", (int) i.si_pid);

141 -#endif

142 - fatal_assert(CLD_EXITED == i.si_code);

143 - tmp.pid = (pid_t)i.si_pid;

144 - while (NULL == (ret_avl = avl_remove_lock(&spawn_outstanding_exec_tree, (avl_t *)&tmp))) {

145 - fprintf(stderr,

146 - "SPAWN: race condition detected, waiting for child process %d to be indexed.\n",

147 - (int)tmp.pid);

148 - (void)sleep_usec(10000); /* 10 msec */

149 - }

150 - exec_info = (struct spawn_execution_info *)ret_avl;

151 - exec_info->exit_status = i.si_status;

152 - enqueue_child_waited_list(exec_info);

153 -

154 - /* wake up event loop */

155 - fatal_assert(0 == uv_async_send(&child_waited_async));

156 - }

157 - }

158 -}

159 -

160 -void spawn_protocol_execute_command(void *handle, char *command_to_run, uint16_t command_length)

161 -{

162 - uv_buf_t *writebuf;

163 - int ret;

164 - avl_t *avl_ret;

165 - struct spawn_execution_info *exec_info;

166 - struct write_context *write_ctx;

167 -

168 - write_ctx = mallocz(sizeof(*write_ctx));

169 - void **data = callocz(2, sizeof(void *));

170 - writebuf = callocz(2, sizeof(uv_buf_t));

171 - data[0] = write_ctx;

172 - data[1] = writebuf;

173 - write_ctx->write_req.data = data;

174 -

175 - command_to_run[command_length] = '\0';

176 -#ifdef SPAWN_DEBUG

177 - fprintf(stderr, "SPAWN: executing command '%s'\n", command_to_run);

178 -#endif

179 - if (netdata_spawn(command_to_run, &write_ctx->spawn_result.exec_pid)) {

180 - fprintf(stderr, "SPAWN: Cannot spawn(\"%s\", \"r\").\n", command_to_run);

181 - write_ctx->spawn_result.exec_pid = 0;

182 - } else { /* successfully spawned command */

183 - write_ctx->spawn_result.exec_run_timestamp = now_realtime_sec();

184 -

185 - /* record it for when the process finishes execution */

186 - exec_info = mallocz(sizeof(*exec_info));

187 - exec_info->handle = handle;

188 - exec_info->pid = write_ctx->spawn_result.exec_pid;

189 - avl_ret = avl_insert_lock(&spawn_outstanding_exec_tree, (avl_t *)exec_info);

190 - fatal_assert(avl_ret == (avl_t *)exec_info);

191 -

192 - /* wake up the thread that blocks waiting for processes to exit */

193 - uv_mutex_lock(&wait_children_mutex);

194 - spawned_processes = 1;

195 - uv_cond_signal(&wait_children_cond);

196 - uv_mutex_unlock(&wait_children_mutex);

197 - }

198 -

199 - write_ctx->header.opcode = SPAWN_PROT_SPAWN_RESULT;

200 - write_ctx->header.handle = handle;

201 - writebuf[0] = uv_buf_init((char *)&write_ctx->header, sizeof(write_ctx->header));

202 - writebuf[1] = uv_buf_init((char *)&write_ctx->spawn_result, sizeof(write_ctx->spawn_result));

203 -#ifdef SPAWN_DEBUG

204 - fprintf(stderr, "SERVER %s SPAWN_PROT_SPAWN_RESULT\n", __func__);

205 -#endif

206 - ret = uv_write(&write_ctx->write_req, (uv_stream_t *)&server_pipe, writebuf, 2, after_pipe_write);

207 - fatal_assert(ret == 0);

208 -}

209 -

210 -static void server_parse_spawn_protocol(unsigned source_len, char *source)

211 -{

212 - unsigned required_len;

213 - struct spawn_prot_header *header;

214 - struct spawn_prot_exec_cmd *payload;

215 - uint16_t command_length;

216 -

217 - while (source_len) {

218 - required_len = sizeof(*header);

219 - if (prot_buffer_len < required_len)

220 - copy_to_prot_buffer(prot_buffer, &prot_buffer_len, required_len - prot_buffer_len, &source, &source_len);

221 - if (prot_buffer_len < required_len)

222 - return; /* Source buffer ran out */

223 -

224 - header = (struct spawn_prot_header *)prot_buffer;

225 - fatal_assert(SPAWN_PROT_EXEC_CMD == header->opcode);

226 - fatal_assert(NULL != header->handle);

227 -

228 - required_len += sizeof(*payload);

229 - if (prot_buffer_len < required_len)

230 - copy_to_prot_buffer(prot_buffer, &prot_buffer_len, required_len - prot_buffer_len, &source, &source_len);

231 - if (prot_buffer_len < required_len)

232 - return; /* Source buffer ran out */

233 -

234 - payload = (struct spawn_prot_exec_cmd *)(header + 1);

235 - command_length = payload->command_length;

236 -

237 - required_len += command_length;

238 - if (unlikely(required_len > MAX_COMMAND_LENGTH - 1)) {

239 - fprintf(stderr, "SPAWN: Ran out of protocol buffer space.\n");

240 - command_length = (MAX_COMMAND_LENGTH - 1) - (sizeof(*header) + sizeof(*payload));

241 - required_len = MAX_COMMAND_LENGTH - 1;

242 - }

243 - if (prot_buffer_len < required_len)

244 - copy_to_prot_buffer(prot_buffer, &prot_buffer_len, required_len - prot_buffer_len, &source, &source_len);

245 - if (prot_buffer_len < required_len)

246 - return; /* Source buffer ran out */

247 -

248 - spawn_protocol_execute_command(header->handle, payload->command_to_run, command_length);

249 - prot_buffer_len = 0;

250 - }

251 -}

252 -

253 -static void on_pipe_read(uv_stream_t *pipe, ssize_t nread, const uv_buf_t *buf)

254 -{

255 - if (0 == nread) {

256 - fprintf(stderr, "SERVER %s: Zero bytes read from spawn pipe.\n", __func__);

257 - } else if (UV_EOF == nread) {

258 - fprintf(stderr, "EOF found in spawn pipe.\n");

259 - } else if (nread < 0) {

260 - fprintf(stderr, "%s: %s\n", __func__, uv_strerror(nread));

261 - }

262 -

263 - if (nread < 0) { /* stop spawn server due to EOF or error */

264 - int error;

265 -

266 - uv_mutex_lock(&wait_children_mutex);

267 - server_shutdown = 1;

268 - spawned_processes = 1;

269 - uv_cond_signal(&wait_children_cond);

270 - uv_mutex_unlock(&wait_children_mutex);

271 -

272 - fprintf(stderr, "Shutting down spawn server event loop.\n");

273 - /* cleanup operations of the event loop */

274 - (void)uv_read_stop((uv_stream_t *) pipe);

275 - uv_close((uv_handle_t *)&server_pipe, NULL);

276 -

277 - error = uv_thread_join(&thread);

278 - if (error) {

279 - fprintf(stderr, "uv_thread_create(): %s", uv_strerror(error));

280 - }

281 - /* After joining it is safe to destroy child_waited_async */

282 - uv_close((uv_handle_t *)&child_waited_async, NULL);

283 - } else if (nread) {

284 -#ifdef SPAWN_DEBUG

285 - fprintf(stderr, "SERVER %s nread %u\n", __func__, (unsigned)nread);

286 -#endif

287 - server_parse_spawn_protocol(nread, buf->base);

288 - }

289 - if (buf && buf->len) {

290 - freez(buf->base);

291 - }

292 -}

293 -

294 -static void on_read_alloc(uv_handle_t *handle,

295 - size_t suggested_size,

296 - uv_buf_t* buf)

297 -{

298 - (void)handle;

299 - buf->base = mallocz(suggested_size);

300 - buf->len = suggested_size;

301 -}

302 -

303 -static void ignore_signal_handler(int signo) {

304 - /*

305 - * By having a signal handler we allow spawned processes to reset default signal dispositions. Setting SIG_IGN

306 - * would be inherited by the spawned children which is not desirable.

307 - */

308 - (void)signo;

309 -}

310 -

311 -void spawn_server(void)

312 -{

313 - int error;

314 -

315 - // initialize the system clocks

316 - clocks_init();

317 -

318 - // close all open file descriptors, except the standard ones

319 - // the caller may have left open files (lxc-attach has this issue)

320 - for_each_open_fd(OPEN_FD_ACTION_CLOSE, OPEN_FD_EXCLUDE_STDIN | OPEN_FD_EXCLUDE_STDOUT | OPEN_FD_EXCLUDE_STDERR);

321 -

322 - // Have the libuv IPC pipe be closed when forking child processes

323 - (void) fcntl(0, F_SETFD, FD_CLOEXEC);

324 - fprintf(stderr, "Spawn server is up.\n");

325 -

326 - // Define signals we want to ignore

327 - struct sigaction sa;

328 - int signals_to_ignore[] = {SIGPIPE, SIGINT, SIGQUIT, SIGTERM, SIGHUP, SIGUSR1, SIGUSR2, SIGBUS, SIGCHLD};

329 - unsigned ignore_length = sizeof(signals_to_ignore) / sizeof(signals_to_ignore[0]);

330 -

331 - unsigned i;

332 - for (i = 0; i < ignore_length ; ++i) {

333 - sa.sa_flags = 0;

334 - sigemptyset(&sa.sa_mask);

335 - sa.sa_handler = ignore_signal_handler;

336 - if(sigaction(signals_to_ignore[i], &sa, NULL) == -1)

337 - fprintf(stderr, "SPAWN: Failed to change signal handler for signal: %d.\n", signals_to_ignore[i]);

338 - }

339 -

340 - signals_unblock();

341 -

342 - loop = uv_default_loop();

343 - loop->data = NULL;

344 -

345 - error = uv_pipe_init(loop, &server_pipe, 1);

346 - if (error) {

347 - fprintf(stderr, "uv_pipe_init(): %s\n", uv_strerror(error));

348 - exit(error);

349 - }

350 - fatal_assert(server_pipe.ipc);

351 -

352 - error = uv_pipe_open(&server_pipe, 0 /* UV_STDIN_FD */);

353 - if (error) {

354 - fprintf(stderr, "uv_pipe_open(): %s\n", uv_strerror(error));

355 - exit(error);

356 - }

357 - avl_init_lock(&spawn_outstanding_exec_tree, spawn_exec_compare);

358 -

359 - spawned_processes = 0;

360 - fatal_assert(0 == uv_cond_init(&wait_children_cond));

361 - fatal_assert(0 == uv_mutex_init(&wait_children_mutex));

362 - child_waited_list = NULL;

363 - error = uv_async_init(loop, &child_waited_async, child_waited_async_cb);

364 - if (error) {

365 - fprintf(stderr, "uv_async_init(): %s\n", uv_strerror(error));

366 - exit(error);

367 - }

368 -

369 - error = uv_thread_create(&thread, wait_children, NULL);

370 - if (error) {

371 - fprintf(stderr, "uv_thread_create(): %s\n", uv_strerror(error));

372 - exit(error);

373 - }

374 -

375 - prot_buffer_len = 0;

376 - error = uv_read_start((uv_stream_t *)&server_pipe, on_read_alloc, on_pipe_read);

377 - fatal_assert(error == 0);

378 -

379 - while (!server_shutdown) {

380 - uv_run(loop, UV_RUN_DEFAULT);

381 - }

382 - fprintf(stderr, "Shutting down spawn server loop complete.\n");

383 - fatal_assert(0 == uv_loop_close(loop));

384 -

385 - exit(0);

386 -}

src/streaming/receiver.c

+1 -1

@@ -70,7 +70,7 @@ static inline int read_stream(struct receiver_state *r, char* buffer, size_t siz

70 ssize_t bytes_read;

72 do {

73 - errno = 0;

73 + errno_clear();

75 switch(wait_on_socket_or_cancel_with_timeout(

76 #ifdef ENABLE_HTTPS

src/streaming/rrdpush.c

+1 -1

@@ -54,7 +54,7 @@ char *netdata_ssl_ca_file = NULL;

54 #endif

56 static void load_stream_conf() {

57 - errno = 0;

57 + errno_clear();

58 char *filename = strdupz_path_subpath(netdata_configured_user_config_dir, "stream.conf");

59 if(!appconfig_load(&stream_config, filename, 0, NULL)) {

60 nd_log_daemon(NDLP_NOTICE, "CONFIG: cannot load user config '%s'. Will try stock config.", filename);

src/streaming/sender.c

+1 -1

@@ -1894,7 +1894,7 @@ void *rrdpush_sender_thread(void *ptr) {

1894 // protection from overflow

1895 if(unlikely(s->flags & SENDER_FLAG_OVERFLOW)) {

1896 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW);

1897 - errno = 0;

1897 + errno_clear();

1898 netdata_log_error("STREAM %s [send to %s]: buffer full (allocated %zu bytes) after sending %zu bytes. Restarting connection",

1899 rrdhost_hostname(s->host), s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection);

1900 rrdpush_sender_thread_close_socket(s->host);

src/web/api/queries/query.c

+1 -1

@@ -691,7 +691,7 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_TIME_GROUPING group_method)

691 }

692 }

693 if(!found) {

694 - errno = 0;

694 + errno_clear();

695 internal_error(true, "QUERY: grouping method %u not found. Using 'average'", (unsigned int)group_method);

696 r->time_grouping.create = tg_average_create;

697 r->time_grouping.reset = tg_average_reset;

src/web/server/web_client.c

+1 -1

@@ -1855,7 +1855,7 @@ ssize_t web_client_receive(struct web_client *w)

1855 // do we have any space for more data?

1856 buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE);

1857

1858 - errno = 0;

1858 + errno_clear();

1859

1860 #ifdef ENABLE_HTTPS

1861 if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {