@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3eea74889f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[misc][distributed] use 127.0.0.1 for single-node (#5619)

youkaichao committed Jun 19, 2024 at 01:05 UTC 3eea74889fe29534808bae41fca251e0e74c0962
2 files changed +15 -2
vllm/executor/multiproc_gpu_executor.py
+5 -2
@@ -10,7 +10,7 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
10 from vllm.logger import init_logger
11 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 from vllm.utils import (cuda_device_count_stateless,
13 - get_distributed_init_method, get_ip, get_open_port,
13 + get_distributed_init_method, get_open_port,
14 get_vllm_instance_id, make_async)
15
16 logger = init_logger(__name__)
@@ -37,8 +37,11 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
37 assert world_size <= cuda_device_count_stateless(), (
38 "please set tensor_parallel_size to less than max local gpu count")
39
40 + # Multiprocessing-based executor does not support multi-node setting.
41 + # Since it only works for single node, we can use the loopback address
42 + # 127.0.0.1 for communication.
43 distributed_init_method = get_distributed_init_method(
41 - get_ip(), get_open_port())
44 + "127.0.0.1", get_open_port())
45
46 if world_size == 1:
47 self.workers = []
vllm/executor/ray_gpu_executor.py
+10
@@ -161,6 +161,16 @@ class RayGPUExecutor(DistributedGPUExecutor):
161 self._run_workers("update_environment_variables",
162 all_args=all_args_to_update_environment_variables)
163
164 + if len(node_gpus) == 1:
165 + # in single node case, we don't need to get the IP address.
166 + # the loopback address is sufficient
167 + # NOTE: a node may have several IP addresses, one for each
168 + # network interface. `get_ip()` might return any of them,
169 + # while they might not work for communication inside the node
170 + # if the network setup is complicated. Using the loopback address
171 + # solves this issue, as it always works for communication inside
172 + # the node.
173 + driver_ip = "127.0.0.1"
174 distributed_init_method = get_distributed_init_method(
175 driver_ip, get_open_port())
176