@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 546a97ef69
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc]: allow user to specify port in distributed setting (#4914)

Wenwei Zhang committed May 21, 2024 at 01:45 UTC 546a97ef691f242c899a5e0906d0e75f42694e95
2 files changed +10
vllm/envs.py
+7
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
3
4 if TYPE_CHECKING:
5 VLLM_HOST_IP: str = ""
6 + VLLM_PORT: Optional[int] = None
7 VLLM_USE_MODELSCOPE: bool = False
8 VLLM_INSTANCE_ID: Optional[str] = None
9 VLLM_NCCL_SO_PATH: Optional[str] = None
@@ -96,6 +97,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
97 'VLLM_HOST_IP':
98 lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
99
100 + # used in distributed environment to manually set the communication port
101 + # '0' is used to make mypy happy
102 + 'VLLM_PORT':
103 + lambda: int(os.getenv('VLLM_PORT', '0'))
104 + if 'VLLM_PORT' in os.environ else None,
105 +
106 # If true, will load models from ModelScope instead of Hugging Face Hub.
107 # note that the value is true or false, not numbers
108 "VLLM_USE_MODELSCOPE":
vllm/utils.py
+3
@@ -282,6 +282,9 @@ def get_distributed_init_method(ip: str, port: int) -> str:
282
283
284 def get_open_port() -> int:
285 + port = envs.VLLM_PORT
286 + if port is not None:
287 + return port
288 # try ipv4
289 try:
290 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: