A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] Allow IPv6 in VLLM_HOST_IP with zmq (#8575)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Sep 23, 2024 at 15:15 UTC
b05f5c9238c3e0c3a98080b4ffc90acfa33f9e1f
2 files changed
+15
-1
vllm/distributed/device_communicators/shm_broadcast.py
+6
-1
@@ -9,11 +9,12 @@ from unittest.mock import patch
9
import torch
10
import torch.distributed as dist
11
from torch.distributed import ProcessGroup
12
+from zmq import IPV6 # type: ignore
13
from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore
14
15
import vllm.envs as envs
16
from vllm.logger import init_logger
16
-from vllm.utils import get_ip, get_open_port
17
+from vllm.utils import get_ip, get_open_port, is_valid_ipv6_address
18
19
VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
20
@@ -214,6 +215,8 @@ class MessageQueue:
215
self.remote_socket = context.socket(XPUB)
216
self.remote_socket.setsockopt(XPUB_VERBOSE, True)
217
remote_subscribe_port = get_open_port()
218
+ if is_valid_ipv6_address(connect_ip):
219
+ self.remote_socket.setsockopt(IPV6, 1)
220
socket_addr = f"tcp://*:{remote_subscribe_port}"
221
self.remote_socket.bind(socket_addr)
222
@@ -274,6 +277,8 @@ class MessageQueue:
277
278
self.remote_socket = context.socket(SUB)
279
self.remote_socket.setsockopt_string(SUBSCRIBE, "")
280
+ if is_valid_ipv6_address(handle.connect_ip):
281
+ self.remote_socket.setsockopt(IPV6, 1)
282
socket_addr = f"tcp://{handle.connect_ip}:{handle.remote_subscribe_port}"
283
logger.debug("Connecting to %s", socket_addr)
284
self.remote_socket.connect(socket_addr)
vllm/utils.py
+9
@@ -5,6 +5,7 @@ import datetime
5
import enum
6
import gc
7
import inspect
8
+import ipaddress
9
import os
10
import random
11
import socket
@@ -533,6 +534,14 @@ def get_ip() -> str:
534
return "0.0.0.0"
535
536
537
+def is_valid_ipv6_address(address: str) -> bool:
538
+ try:
539
+ ipaddress.IPv6Address(address)
540
+ return True
541
+ except ValueError:
542
+ return False
543
+
544
+
545
def get_distributed_init_method(ip: str, port: int) -> str:
546
# Brackets are not permitted in ipv4 addresses,
547
# see https://github.com/python/cpython/issues/103848