A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Misc] add HOST_IP env var (#3419)
Co-authored-by: Simon Mo <simon.mo@hey.com>
youkaichao committed
Mar 14, 2024 at 21:32 UTC
b522c4476fcdaee254fe40fefb354a4908fccac5
1 file changed
+23
-3
vllm/utils.py
+23
-3
@@ -21,6 +21,7 @@ from collections import OrderedDict
21
from typing import Any, Hashable, Optional
22
23
from vllm.logger import init_logger
24
+import warnings
25
26
T = TypeVar("T")
27
logger = init_logger(__name__)
@@ -172,16 +173,35 @@ def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
173
174
175
def get_ip() -> str:
176
+ host_ip = os.environ.get("HOST_IP")
177
+ if host_ip:
178
+ return host_ip
179
+
180
+ # IP is not set, try to get it from the network interface
181
+
182
# try ipv4
183
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
184
try:
185
s.connect(("8.8.8.8", 80)) # Doesn't need to be reachable
186
return s.getsockname()[0]
180
- except OSError:
181
- # try ipv6
187
+ except Exception:
188
+ pass
189
+
190
+ # try ipv6
191
+ try:
192
s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
183
- s.connect(("dns.google", 80))
193
+ # Google's public DNS server, see
194
+ # https://developers.google.com/speed/public-dns/docs/using#addresses
195
+ s.connect(("2001:4860:4860::8888", 80)) # Doesn't need to be reachable
196
return s.getsockname()[0]
197
+ except Exception:
198
+ pass
199
+
200
+ warnings.warn(
201
+ "Failed to get the IP address, using 0.0.0.0 by default."
202
+ "The value can be set by the environment variable HOST_IP.",
203
+ stacklevel=2)
204
+ return "0.0.0.0"
205
206
207
def get_distributed_init_method(ip: str, port: int) -> str: