@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b522c4476f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] add HOST_IP env var (#3419)

Co-authored-by: Simon Mo <simon.mo@hey.com>

youkaichao committed Mar 14, 2024 at 21:32 UTC b522c4476fcdaee254fe40fefb354a4908fccac5
1 file changed +23 -3
vllm/utils.py
+23 -3
@@ -21,6 +21,7 @@ from collections import OrderedDict
21 from typing import Any, Hashable, Optional
22
23 from vllm.logger import init_logger
24 +import warnings
25
26 T = TypeVar("T")
27 logger = init_logger(__name__)
@@ -172,16 +173,35 @@ def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
173
174
175 def get_ip() -> str:
176 + host_ip = os.environ.get("HOST_IP")
177 + if host_ip:
178 + return host_ip
179 +
180 + # IP is not set, try to get it from the network interface
181 +
182 # try ipv4
183 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
184 try:
185 s.connect(("8.8.8.8", 80)) # Doesn't need to be reachable
186 return s.getsockname()[0]
180 - except OSError:
181 - # try ipv6
187 + except Exception:
188 + pass
189 +
190 + # try ipv6
191 + try:
192 s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
183 - s.connect(("dns.google", 80))
193 + # Google's public DNS server, see
194 + # https://developers.google.com/speed/public-dns/docs/using#addresses
195 + s.connect(("2001:4860:4860::8888", 80)) # Doesn't need to be reachable
196 return s.getsockname()[0]
197 + except Exception:
198 + pass
199 +
200 + warnings.warn(
201 + "Failed to get the IP address, using 0.0.0.0 by default."
202 + "The value can be set by the environment variable HOST_IP.",
203 + stacklevel=2)
204 + return "0.0.0.0"
205
206
207 def get_distributed_init_method(ip: str, port: int) -> str: