A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc][Utils] allow get_open_port to be called for multiple times (#5333)
youkaichao committed
Jun 6, 2024 at 22:15 UTC
388596c91437a51d428a447594e9faec340c29b2
3 files changed
+27
-2
tests/test_utils.py
+15
-1
@@ -1,11 +1,13 @@
1
import asyncio
2
+import os
3
+import socket
4
import sys
5
from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
6
Tuple, TypeVar)
7
8
import pytest
9
8
-from vllm.utils import deprecate_kwargs, merge_async_iterators
10
+from vllm.utils import deprecate_kwargs, get_open_port, merge_async_iterators
11
12
from .utils import error_on_warning
13
@@ -116,3 +118,15 @@ def test_deprecate_kwargs_additional_message():
118
119
with pytest.warns(DeprecationWarning, match="abcd"):
120
dummy(old_arg=1)
121
+
122
+
123
+def test_get_open_port():
124
+ os.environ["VLLM_PORT"] = "5678"
125
+ # make sure we can get multiple ports, even if the env var is set
126
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s1:
127
+ s1.bind(("localhost", get_open_port()))
128
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s2:
129
+ s2.bind(("localhost", get_open_port()))
130
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s3:
131
+ s3.bind(("localhost", get_open_port()))
132
+ os.environ.pop("VLLM_PORT")
vllm/envs.py
+3
@@ -99,6 +99,9 @@ environment_variables: Dict[str, Callable[[], Any]] = {
99
lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
100
101
# used in distributed environment to manually set the communication port
102
+ # Note: if VLLM_PORT is set, and some code asks for multiple ports, the
103
+ # VLLM_PORT will be used as the first port, and the rest will be generated
104
+ # by incrementing the VLLM_PORT value.
105
# '0' is used to make mypy happy
106
'VLLM_PORT':
107
lambda: int(os.getenv('VLLM_PORT', '0'))
vllm/utils.py
+9
-1
@@ -289,7 +289,15 @@ def get_distributed_init_method(ip: str, port: int) -> str:
289
def get_open_port() -> int:
290
port = envs.VLLM_PORT
291
if port is not None:
292
- return port
292
+ while True:
293
+ try:
294
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
295
+ s.bind(("", port))
296
+ return port
297
+ except OSError:
298
+ port += 1 # Increment port number if already in use
299
+ logger.info("Port %d is already in use, trying port %d",
300
+ port - 1, port)
301
# try ipv4
302
try:
303
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: