@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b05f5c9238
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] Allow IPv6 in VLLM_HOST_IP with zmq (#8575)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Sep 23, 2024 at 15:15 UTC b05f5c9238c3e0c3a98080b4ffc90acfa33f9e1f
2 files changed +15 -1
vllm/distributed/device_communicators/shm_broadcast.py
+6 -1
@@ -9,11 +9,12 @@ from unittest.mock import patch
9 import torch
10 import torch.distributed as dist
11 from torch.distributed import ProcessGroup
12 +from zmq import IPV6 # type: ignore
13 from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore
14
15 import vllm.envs as envs
16 from vllm.logger import init_logger
16 -from vllm.utils import get_ip, get_open_port
17 +from vllm.utils import get_ip, get_open_port, is_valid_ipv6_address
18
19 VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
20
@@ -214,6 +215,8 @@ class MessageQueue:
215 self.remote_socket = context.socket(XPUB)
216 self.remote_socket.setsockopt(XPUB_VERBOSE, True)
217 remote_subscribe_port = get_open_port()
218 + if is_valid_ipv6_address(connect_ip):
219 + self.remote_socket.setsockopt(IPV6, 1)
220 socket_addr = f"tcp://*:{remote_subscribe_port}"
221 self.remote_socket.bind(socket_addr)
222
@@ -274,6 +277,8 @@ class MessageQueue:
277
278 self.remote_socket = context.socket(SUB)
279 self.remote_socket.setsockopt_string(SUBSCRIBE, "")
280 + if is_valid_ipv6_address(handle.connect_ip):
281 + self.remote_socket.setsockopt(IPV6, 1)
282 socket_addr = f"tcp://{handle.connect_ip}:{handle.remote_subscribe_port}"
283 logger.debug("Connecting to %s", socket_addr)
284 self.remote_socket.connect(socket_addr)
vllm/utils.py
+9
@@ -5,6 +5,7 @@ import datetime
5 import enum
6 import gc
7 import inspect
8 +import ipaddress
9 import os
10 import random
11 import socket
@@ -533,6 +534,14 @@ def get_ip() -> str:
534 return "0.0.0.0"
535
536
537 +def is_valid_ipv6_address(address: str) -> bool:
538 + try:
539 + ipaddress.IPv6Address(address)
540 + return True
541 + except ValueError:
542 + return False
543 +
544 +
545 def get_distributed_init_method(ip: str, port: int) -> str:
546 # Brackets are not permitted in ipv4 addresses,
547 # see https://github.com/python/cpython/issues/103848