A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[misc][distributed] use 127.0.0.1 for single-node (#5619)
youkaichao committed
Jun 19, 2024 at 01:05 UTC
3eea74889fe29534808bae41fca251e0e74c0962
2 files changed
+15
-2
vllm/executor/multiproc_gpu_executor.py
+5
-2
@@ -10,7 +10,7 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
10
from vllm.logger import init_logger
11
from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
from vllm.utils import (cuda_device_count_stateless,
13
- get_distributed_init_method, get_ip, get_open_port,
13
+ get_distributed_init_method, get_open_port,
14
get_vllm_instance_id, make_async)
15
16
logger = init_logger(__name__)
@@ -37,8 +37,11 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
37
assert world_size <= cuda_device_count_stateless(), (
38
"please set tensor_parallel_size to less than max local gpu count")
39
40
+ # Multiprocessing-based executor does not support multi-node setting.
41
+ # Since it only works for single node, we can use the loopback address
42
+ # 127.0.0.1 for communication.
43
distributed_init_method = get_distributed_init_method(
41
- get_ip(), get_open_port())
44
+ "127.0.0.1", get_open_port())
45
46
if world_size == 1:
47
self.workers = []
vllm/executor/ray_gpu_executor.py
+10
@@ -161,6 +161,16 @@ class RayGPUExecutor(DistributedGPUExecutor):
161
self._run_workers("update_environment_variables",
162
all_args=all_args_to_update_environment_variables)
163
164
+ if len(node_gpus) == 1:
165
+ # in single node case, we don't need to get the IP address.
166
+ # the loopback address is sufficient
167
+ # NOTE: a node may have several IP addresses, one for each
168
+ # network interface. `get_ip()` might return any of them,
169
+ # while they might not work for communication inside the node
170
+ # if the network setup is complicated. Using the loopback address
171
+ # solves this issue, as it always works for communication inside
172
+ # the node.
173
+ driver_ip = "127.0.0.1"
174
distributed_init_method = get_distributed_init_method(
175
driver_ip, get_open_port())
176