A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[distributed][misc] error on same VLLM_HOST_IP setting (#7756)
youkaichao committed
Aug 21, 2024 at 16:25 UTC
7eebe8ccaa8bb9c37d59d00cbedcd5e67308acfe
2 files changed
+17
-1
vllm/envs.py
+4
-1
@@ -137,7 +137,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
137
os.path.join(get_default_cache_root(), "vllm"),
138
)),
139
140
- # used in distributed environment to determine the master address
140
+ # used in distributed environment to determine the ip address
141
+ # of the current node, when the node has multiple network interfaces.
142
+ # If you are using multi-node inference, you should set this differently
143
+ # on each node.
144
'VLLM_HOST_IP':
145
lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
146
vllm/executor/ray_gpu_executor.py
+13
@@ -218,6 +218,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
218
for node_id, gpu_ids in node_gpus.items():
219
node_gpus[node_id] = sorted(gpu_ids)
220
221
+ all_ips = set(worker_ips + [driver_ip])
222
+ n_ips = len(all_ips)
223
+ n_nodes = len(node_workers)
224
+
225
+ if n_nodes != n_ips:
226
+ raise RuntimeError(
227
+ f"Every node should have a unique IP address. Got {n_nodes}"
228
+ f" nodes with node ids {list(node_workers.keys())} and "
229
+ f"{n_ips} unique IP addresses {all_ips}. Please check your"
230
+ " network configuration. If you set `VLLM_HOST_IP` or "
231
+ "`HOST_IP` environment variable, make sure it is unique for"
232
+ " each node.")
233
+
234
VLLM_INSTANCE_ID = get_vllm_instance_id()
235
236
# Set environment variables for the driver and workers.