@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7eebe8ccaa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[distributed][misc] error on same VLLM_HOST_IP setting (#7756)

youkaichao committed Aug 21, 2024 at 16:25 UTC 7eebe8ccaa8bb9c37d59d00cbedcd5e67308acfe
2 files changed +17 -1
vllm/envs.py
+4 -1
@@ -137,7 +137,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
137 os.path.join(get_default_cache_root(), "vllm"),
138 )),
139
140 - # used in distributed environment to determine the master address
140 + # used in distributed environment to determine the ip address
141 + # of the current node, when the node has multiple network interfaces.
142 + # If you are using multi-node inference, you should set this differently
143 + # on each node.
144 'VLLM_HOST_IP':
145 lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
146
vllm/executor/ray_gpu_executor.py
+13
@@ -218,6 +218,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
218 for node_id, gpu_ids in node_gpus.items():
219 node_gpus[node_id] = sorted(gpu_ids)
220
221 + all_ips = set(worker_ips + [driver_ip])
222 + n_ips = len(all_ips)
223 + n_nodes = len(node_workers)
224 +
225 + if n_nodes != n_ips:
226 + raise RuntimeError(
227 + f"Every node should have a unique IP address. Got {n_nodes}"
228 + f" nodes with node ids {list(node_workers.keys())} and "
229 + f"{n_ips} unique IP addresses {all_ips}. Please check your"
230 + " network configuration. If you set `VLLM_HOST_IP` or "
231 + "`HOST_IP` environment variable, make sure it is unique for"
232 + " each node.")
233 +
234 VLLM_INSTANCE_ID = get_vllm_instance_id()
235
236 # Set environment variables for the driver and workers.