A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc]: allow user to specify port in distributed setting (#4914)
Wenwei Zhang committed
May 21, 2024 at 01:45 UTC
546a97ef691f242c899a5e0906d0e75f42694e95
2 files changed
+10
vllm/envs.py
+7
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
3
4
if TYPE_CHECKING:
5
VLLM_HOST_IP: str = ""
6
+ VLLM_PORT: Optional[int] = None
7
VLLM_USE_MODELSCOPE: bool = False
8
VLLM_INSTANCE_ID: Optional[str] = None
9
VLLM_NCCL_SO_PATH: Optional[str] = None
@@ -96,6 +97,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
97
'VLLM_HOST_IP':
98
lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
99
100
+ # used in distributed environment to manually set the communication port
101
+ # '0' is used to make mypy happy
102
+ 'VLLM_PORT':
103
+ lambda: int(os.getenv('VLLM_PORT', '0'))
104
+ if 'VLLM_PORT' in os.environ else None,
105
+
106
# If true, will load models from ModelScope instead of Hugging Face Hub.
107
# note that the value is true or false, not numbers
108
"VLLM_USE_MODELSCOPE":
vllm/utils.py
+3
@@ -282,6 +282,9 @@ def get_distributed_init_method(ip: str, port: int) -> str:
282
283
284
def get_open_port() -> int:
285
+ port = envs.VLLM_PORT
286
+ if port is not None:
287
+ return port
288
# try ipv4
289
try:
290
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: