master
cpp 536 lines 18.7 KB
Raw
1 // Copyright (C) Microsoft Corporation. All rights reserved.
2 #include "common.h"
3 #include <memory>
4 #include <string>
5 #include <string_view>
6 #include <vector>
7 #include <iostream>
8
9 #include <libgen.h>
10 #include <sys/epoll.h>
11 #include <sys/socket.h>
12 #include <netinet/ip.h>
13 #include <sys/syscall.h>
14 #include <linux/unistd.h>
15 #include <linux/sock_diag.h>
16 #include <linux/inet_diag.h>
17 #include <lxwil.h>
18 #include <linux/if_tun.h>
19
20 #include "util.h"
21 #include "SocketChannel.h"
22 #include "GnsPortTracker.h"
23 #include "SecCompDispatcher.h"
24 #include "seccomp_defs.h"
25 #include "CommandLine.h"
26 #include "NetlinkChannel.h"
27 #include "NetlinkTransactionError.h"
28
29 #define TCP_LISTEN 10
30
31 namespace {
32
33 std::vector<sockaddr_storage> QueryListeningSockets(NetlinkChannel& channel)
34 {
35 std::vector<sockaddr_storage> sockets{};
36 try
37 {
38 inet_diag_req_v2 message{};
39 message.sdiag_protocol = IPPROTO_TCP;
40 message.idiag_states = (1 << TCP_LISTEN);
41
42 auto onMessage = [&](const NetlinkResponse& response) {
43 for (const auto& e : response.Messages<inet_diag_msg>(SOCK_DIAG_BY_FAMILY))
44 {
45 const auto* payload = e.Payload();
46 sockaddr_storage sock{};
47
48 if (payload->idiag_family == AF_INET)
49 {
50 auto* ipv4 = reinterpret_cast<sockaddr_in*>(&sock);
51 ipv4->sin_family = AF_INET;
52 ipv4->sin_addr.s_addr = payload->id.idiag_src[0];
53 ipv4->sin_port = payload->id.idiag_sport;
54 }
55 else if (payload->idiag_family == AF_INET6)
56 {
57 auto* ipv6 = reinterpret_cast<sockaddr_in6*>(&sock);
58 ipv6->sin6_family = AF_INET6;
59 static_assert(sizeof(ipv6->sin6_addr.s6_addr32) == sizeof(payload->id.idiag_src));
60 memcpy(ipv6->sin6_addr.s6_addr32, payload->id.idiag_src, sizeof(ipv6->sin6_addr.s6_addr32));
61 ipv6->sin6_port = payload->id.idiag_sport;
62 }
63
64 sockets.emplace_back(sock);
65 }
66 };
67
68 // Query IPv4 listening sockets.
69 {
70 message.sdiag_family = AF_INET;
71 auto transaction = channel.CreateTransaction(message, SOCK_DIAG_BY_FAMILY, NLM_F_DUMP);
72 transaction.Execute(onMessage);
73 }
74
75 // Query IPv6 listening sockets.
76 {
77 message.sdiag_family = AF_INET6;
78 auto transaction = channel.CreateTransaction(message, SOCK_DIAG_BY_FAMILY, NLM_F_DUMP);
79 transaction.Execute(onMessage);
80 }
81 }
82 catch (const NetlinkTransactionError& e)
83 {
84 // Log but don't fail - network state might be temporarily unavailable
85 LOG_ERROR("Failed to query listening sockets via sock_diag: {}", e.what());
86 }
87
88 return sockets;
89 }
90
91 int SendRelayListenerSocket(wsl::shared::SocketChannel& channel, int hvSocketPort)
92 try
93 {
94 LX_GNS_SET_PORT_LISTENER message{};
95 message.Header.MessageType = LxGnsMessageSetPortListener;
96 message.Header.MessageSize = sizeof(message);
97 message.HvSocketPort = hvSocketPort;
98
99 channel.SendMessage(message);
100
101 return 0;
102 }
103 CATCH_RETURN_ERRNO();
104
105 LX_GNS_PORT_LISTENER_RELAY SockToRelayMessage(const sockaddr_storage& sock)
106 {
107 LX_GNS_PORT_LISTENER_RELAY message{};
108 message.Header.MessageSize = sizeof(message);
109 message.Family = sock.ss_family;
110 if (sock.ss_family == AF_INET)
111 {
112 auto ipv4 = reinterpret_cast<const sockaddr_in*>(&sock);
113 message.Address[0] = ipv4->sin_addr.s_addr;
114 message.Port = ntohs(ipv4->sin_port);
115 }
116 else if (sock.ss_family == AF_INET6)
117 {
118 auto ipv6 = reinterpret_cast<const sockaddr_in6*>(&sock);
119 message.Port = ntohs(ipv6->sin6_port);
120 memcpy(message.Address, ipv6->sin6_addr.__in6_union.__s6_addr, sizeof(message.Address));
121 }
122 return message;
123 }
124
125 int StartHostListener(wsl::shared::SocketChannel& channel, const sockaddr_storage& sock)
126 try
127 {
128 auto message = SockToRelayMessage(sock);
129 message.Header.MessageType = LxGnsMessagePortListenerRelayStart;
130 auto transaction = channel.StartTransaction();
131 transaction.Send(message);
132
133 return 0;
134 }
135 CATCH_RETURN_ERRNO();
136
137 int StopHostListener(wsl::shared::SocketChannel& channel, const sockaddr_storage& sock)
138 try
139 {
140 auto message = SockToRelayMessage(sock);
141 message.Header.MessageType = LxGnsMessagePortListenerRelayStop;
142 auto transaction = channel.StartTransaction();
143 transaction.Send(message);
144
145 return 0;
146 }
147 CATCH_RETURN_ERRNO();
148
149 bool IsSameSockAddr(const sockaddr_storage& left, const sockaddr_storage& right)
150 {
151 if (left.ss_family != right.ss_family)
152 {
153 return false;
154 }
155
156 if (left.ss_family == AF_INET)
157 {
158 auto leftIpv4 = reinterpret_cast<const sockaddr_in*>(&left);
159 auto rightIpv4 = reinterpret_cast<const sockaddr_in*>(&right);
160 return (leftIpv4->sin_addr.s_addr == rightIpv4->sin_addr.s_addr && leftIpv4->sin_port == rightIpv4->sin_port);
161 }
162 else if (left.ss_family == AF_INET6)
163 {
164 auto leftIpv6 = reinterpret_cast<const sockaddr_in6*>(&left);
165 auto rightIpv6 = reinterpret_cast<const sockaddr_in6*>(&right);
166 return (leftIpv6->sin6_port == rightIpv6->sin6_port && memcmp(&leftIpv6->sin6_addr, &rightIpv6->sin6_addr, sizeof(in6_addr)) == 0);
167 }
168
169 FATAL_ERROR("Unrecognized socket family {}", left.ss_family);
170 return false;
171 }
172
173 // Monitor listening TCP sockets using sock_diag netlink interface.
174 int MonitorListeningSockets(wsl::shared::SocketChannel& channel)
175 {
176 NetlinkChannel netlinkChannel(SOCK_RAW, NETLINK_SOCK_DIAG);
177 std::vector<sockaddr_storage> relays{};
178 int result = 0;
179
180 for (;;)
181 {
182 auto sockets = QueryListeningSockets(netlinkChannel);
183
184 // Stop any relays that no longer match listening ports.
185 std::erase_if(relays, [&](const auto& entry) {
186 auto found =
187 std::find_if(sockets.begin(), sockets.end(), [&](const auto& socket) { return IsSameSockAddr(entry, socket); });
188
189 bool remove = (found == sockets.end());
190 if (remove)
191 {
192 if (StopHostListener(channel, entry) < 0)
193 {
194 result = -1;
195 }
196 }
197
198 return remove;
199 });
200
201 // Create relays for any new ports.
202 std::for_each(sockets.begin(), sockets.end(), [&](const auto& socket) {
203 auto found =
204 std::find_if(relays.begin(), relays.end(), [&](const auto& entry) { return IsSameSockAddr(entry, socket); });
205
206 if (found == relays.end())
207 {
208 if (StartHostListener(channel, socket) < 0)
209 {
210 result = -1;
211 }
212 else
213 {
214 relays.push_back(socket);
215 }
216 }
217 });
218
219 // Ensure all start / stop operations were successful.
220 if (result < 0)
221 {
222 break;
223 }
224
225 // Sleep before scanning again.
226 std::this_thread::sleep_for(std::chrono::seconds(1));
227 }
228
229 return result;
230 }
231 } // namespace
232
233 void RunLocalHostRelay(sockaddr_vm hvSocketAddress, int listenSocket)
234 {
235 pollfd pollDescriptors[] = {{listenSocket, POLLIN}};
236 for (;;)
237 {
238 int result = poll(pollDescriptors, COUNT_OF(pollDescriptors), -1);
239 if (result < 0)
240 {
241 LOG_ERROR("poll failed {}", errno);
242 return;
243 }
244
245 if ((pollDescriptors[0].revents & POLLIN) == 0)
246 {
247 LOG_ERROR("unexpected revents {:x}", pollDescriptors[0].revents);
248 return;
249 }
250
251 // Accept a connection and start a relay worker thread.
252 wil::unique_fd relaySocket{UtilAcceptVsock(listenSocket, hvSocketAddress)};
253 THROW_LAST_ERROR_IF(!relaySocket);
254
255 std::thread([relaySocket = std::move(relaySocket)]() {
256 try
257 {
258 // Read a message to determine which TCP port to connect to.
259 std::vector<gsl::byte> buffer(sizeof(LX_INIT_START_SOCKET_RELAY));
260 auto bytesRead = UtilReadBuffer(relaySocket.get(), buffer);
261 if (bytesRead == 0)
262 {
263 return;
264 }
265
266 auto* message = gslhelpers::try_get_struct<LX_INIT_START_SOCKET_RELAY>(gsl::make_span(buffer.data(), bytesRead));
267 THROW_ERRNO_IF(EINVAL, !message || (message->Header.MessageType != LxInitMessageStartSocketRelay));
268
269 // Connect to the actual socket address and set up a relay.
270 //
271 // N.B. During the time setting up the relay the server may have
272 // stopped listening.
273 sockaddr* socketAddress;
274 int socketAddressSize;
275 sockaddr_in sockaddrIn{};
276 sockaddr_in6 sockaddrIn6{};
277
278 if (message->Family == AF_INET)
279 {
280 sockaddrIn.sin_family = AF_INET;
281 sockaddrIn.sin_port = htons(message->Port);
282 sockaddrIn.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
283 socketAddress = reinterpret_cast<sockaddr*>(&sockaddrIn);
284 socketAddressSize = sizeof(sockaddrIn);
285 }
286 else if (message->Family == AF_INET6)
287 {
288 sockaddrIn6.sin6_family = AF_INET6;
289 sockaddrIn6.sin6_port = htons(message->Port);
290 sockaddrIn6.sin6_addr = IN6ADDR_LOOPBACK_INIT;
291 socketAddress = reinterpret_cast<sockaddr*>(&sockaddrIn6);
292 socketAddressSize = sizeof(sockaddrIn6);
293 }
294 else
295 {
296 THROW_ERRNO(EINVAL);
297 }
298
299 wil::unique_fd tcpSocket{socket(socketAddress->sa_family, SOCK_STREAM, IPPROTO_TCP)};
300 THROW_LAST_ERROR_IF(!tcpSocket);
301
302 if (TEMP_FAILURE_RETRY(connect(tcpSocket.get(), socketAddress, socketAddressSize)) < 0)
303 {
304 LOG_ERROR("Failed to connect to port: {}, family: {}, errno: {}", message->Port, message->Family, errno);
305 return;
306 }
307
308 // Resize the buffer to be the requested size.
309 buffer.resize(message->BufferSize);
310
311 // Begin relaying data.
312 int outFd[2] = {tcpSocket.get(), relaySocket.get()};
313 pollfd pollDescriptors[] = {{relaySocket.get(), POLLIN}, {tcpSocket.get(), POLLIN}};
314
315 for (;;)
316 {
317 if ((pollDescriptors[0].fd == -1) || (pollDescriptors[1].fd == -1))
318 {
319 return;
320 }
321
322 THROW_LAST_ERROR_IF(poll(pollDescriptors, COUNT_OF(pollDescriptors), -1) < 0);
323
324 bytesRead = 0;
325 for (int Index = 0; Index < COUNT_OF(pollDescriptors); Index += 1)
326 {
327 if (pollDescriptors[Index].revents & POLLIN)
328 {
329 bytesRead = UtilReadBuffer(pollDescriptors[Index].fd, buffer);
330 if (bytesRead == 0)
331 {
332 pollDescriptors[Index].fd = -1;
333 shutdown(outFd[Index], SHUT_WR);
334 }
335 else if (bytesRead < 0)
336 {
337 return;
338 }
339 else if (UtilWriteBuffer(outFd[Index], buffer.data(), bytesRead) < 0)
340 {
341 return;
342 }
343 }
344 }
345 }
346 }
347 CATCH_LOG()
348 }).detach();
349 }
350
351 return;
352 }
353
354 // Create a thread to monitor for connections to relay.
355 int StartLocalhostRelay(wsl::shared::SocketChannel& channel, int GuestRelayFd, bool ScanForPorts)
356 try
357 {
358 // If the other end of a socket is reset, write will result in EPIPE. Ignore
359 // this signal and just use the write return value.
360 THROW_LAST_ERROR_IF(signal(SIGPIPE, SIG_IGN) == SIG_ERR);
361
362 sockaddr_vm hvSocketAddress = {};
363 socklen_t hvSocketAddressLen = sizeof(hvSocketAddress);
364 if (getsockname(GuestRelayFd, reinterpret_cast<sockaddr*>(&hvSocketAddress), &hvSocketAddressLen) < 0 ||
365 hvSocketAddressLen != sizeof(hvSocketAddress))
366 {
367 LOG_ERROR("Failed to get hvsocket port: {}, {}", errno, hvSocketAddressLen);
368 return -1;
369 }
370
371 wil::unique_fd listenSocket{GuestRelayFd};
372 THROW_LAST_ERROR_IF(!listenSocket);
373
374 // Create a thread to accept incoming connections from the host listener
375 std::thread([hvSocketAddress, listenSocket = std::move(listenSocket)]() {
376 try
377 {
378 RunLocalHostRelay(hvSocketAddress, listenSocket.get());
379 }
380 CATCH_LOG()
381 }).detach();
382
383 if (SendRelayListenerSocket(channel, hvSocketAddress.svm_port) < 0)
384 {
385 LOG_ERROR("Unable to send relay listener socket");
386 return -1;
387 }
388
389 if (ScanForPorts)
390 {
391 return MonitorListeningSockets(channel);
392 }
393
394 return 0;
395 }
396 catch (...)
397 {
398 LOG_CAUGHT_EXCEPTION_MSG("Could not start localhost relay.")
399 return -1;
400 }
401
402 int RunPortTracker(int Argc, char** Argv)
403 {
404 using namespace wsl::shared;
405
406 constexpr auto* Usage = "Usage: localhost " INIT_PORT_TRACKER_FD_ARG
407 " fd"
408 " [" INIT_BPF_FD_ARG
409 " fd]"
410 " [" INIT_NETLINK_FD_ARG
411 " fd]"
412 " [" INIT_PORT_TRACKER_LOCALHOST_RELAY
413 " fd]"
414 " [" INIT_PORT_TRACKER_NETWORKING_MODE_ARG " mode]\n";
415
416 // This is only supported on VM mode.
417 if (!UtilIsUtilityVm())
418 {
419 return -1;
420 }
421
422 // Initialize error and telemetry logging.
423 InitializeLogging(true);
424
425 int BpfFd = -1;
426 int PortTrackerFd = -1;
427 int NetlinkSocketFd = -1;
428 int GuestRelayFd = -1;
429 int NetworkingMode = static_cast<int>(LxMiniInitNetworkingModeNone);
430
431 ArgumentParser parser(Argc, Argv);
432 parser.AddArgument(Integer{BpfFd}, INIT_BPF_FD_ARG);
433 parser.AddArgument(Integer{PortTrackerFd}, INIT_PORT_TRACKER_FD_ARG);
434 parser.AddArgument(Integer{NetlinkSocketFd}, INIT_NETLINK_FD_ARG);
435 parser.AddArgument(Integer{GuestRelayFd}, INIT_PORT_TRACKER_LOCALHOST_RELAY);
436 parser.AddArgument(Integer{NetworkingMode}, INIT_PORT_TRACKER_NETWORKING_MODE_ARG);
437
438 try
439 {
440 parser.Parse();
441 }
442 catch (const wil::ExceptionWithUserMessage& e)
443 {
444 std::cerr << e.what() << "\n" << Usage;
445 return 1;
446 }
447
448 if (NetworkingMode < LxMiniInitNetworkingModeNone || NetworkingMode > LxMiniInitNetworkingModeConsomme)
449 {
450 std::cerr << "Invalid networking mode (" << NetworkingMode << ")\n";
451 return 1;
452 }
453
454 const bool synchronousMode = BpfFd != -1 && NetlinkSocketFd != -1;
455 const bool localhostRelay = GuestRelayFd != -1;
456 auto hvSocketChannel = std::make_shared<wsl::shared::SocketChannel>(wil::unique_fd{PortTrackerFd}, "localhost");
457
458 if (localhostRelay)
459 {
460 // This needs to be the first message sent over the PortTrackerFd channel,
461 // before running the seccomp dispatcher loop.
462 const int ret = StartLocalhostRelay(*hvSocketChannel, GuestRelayFd, !synchronousMode);
463 if (ret < 0)
464 {
465 LOG_ERROR("Failed to start the guest side of the localhost relay");
466 }
467 if (!synchronousMode)
468 {
469 return ret;
470 }
471 }
472
473 if (!synchronousMode)
474 {
475 std::cerr << "either both or none of --bpf-fd and --netlink-socket can be passed\n";
476 return 1;
477 }
478
479 auto channel = NetlinkChannel::FromFd(NetlinkSocketFd);
480
481 auto seccompDispatcher = std::make_shared<SecCompDispatcher>(BpfFd);
482
483 GnsPortTracker portTracker(hvSocketChannel, std::move(channel), seccompDispatcher, static_cast<LX_MINI_INIT_NETWORKING_MODE>(NetworkingMode));
484
485 seccompDispatcher->RegisterHandler(
486 __NR_bind, [&portTracker](seccomp_notif* notification) { return portTracker.ProcessSecCompNotification(notification); });
487
488 // listen() can perform an implicit autobind (assigning an ephemeral port) when called on a
489 // socket that was never explicitly bind()'d. That autobind is otherwise invisible to the
490 // port tracker, so listen() needs to be intercepted the same way bind() is.
491 seccompDispatcher->RegisterHandler(
492 __NR_listen, [&portTracker](seccomp_notif* notification) { return portTracker.ProcessSecCompNotification(notification); });
493
494 #ifdef __x86_64__
495 seccompDispatcher->RegisterHandler(I386_NR_socketcall, [&portTracker](seccomp_notif* notification) {
496 return portTracker.ProcessSecCompNotification(notification);
497 });
498 #else
499 seccompDispatcher->RegisterHandler(ARMV7_NR_bind, [&portTracker](seccomp_notif* notification) {
500 return portTracker.ProcessSecCompNotification(notification);
501 });
502 seccompDispatcher->RegisterHandler(ARMV7_NR_listen, [&portTracker](seccomp_notif* notification) {
503 return portTracker.ProcessSecCompNotification(notification);
504 });
505 #endif
506
507 seccompDispatcher->RegisterHandler(__NR_ioctl, [hvSocketChannel, seccompDispatcher](auto notification) -> int {
508 LX_GNS_TUN_BRIDGE_REQUEST request{};
509 request.Header.MessageType = LxGnsMessageIfStateChangeRequest;
510 request.Header.MessageSize = sizeof(request);
511 auto ifreqMemory =
512 seccompDispatcher->ReadProcessMemory(notification->id, notification->pid, notification->data.args[2], sizeof(ifreq));
513 if (!ifreqMemory.has_value())
514 {
515 return -1;
516 }
517
518 auto& ifRequest = *reinterpret_cast<ifreq*>(ifreqMemory->data());
519 memcpy(request.InterfaceName, ifRequest.ifr_ifrn.ifrn_name, sizeof(request.InterfaceName));
520 request.InterfaceUp = ifRequest.ifr_ifru.ifru_flags & IFF_UP;
521 const auto& reply = hvSocketChannel->Transaction(request);
522
523 return reply.Result;
524 });
525
526 try
527 {
528 portTracker.Run();
529 }
530 catch (const std::exception& e)
531 {
532 std::cerr << "Port tracker exiting with fatal error, " << e.what() << std::endl;
533 }
534
535 return 1;
536 }