@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 388596c914
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc][Utils] allow get_open_port to be called for multiple times (#5333)

youkaichao committed Jun 6, 2024 at 22:15 UTC 388596c91437a51d428a447594e9faec340c29b2
3 files changed +27 -2
tests/test_utils.py
+15 -1
@@ -1,11 +1,13 @@
1 import asyncio
2 +import os
3 +import socket
4 import sys
5 from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
6 Tuple, TypeVar)
7
8 import pytest
9
8 -from vllm.utils import deprecate_kwargs, merge_async_iterators
10 +from vllm.utils import deprecate_kwargs, get_open_port, merge_async_iterators
11
12 from .utils import error_on_warning
13
@@ -116,3 +118,15 @@ def test_deprecate_kwargs_additional_message():
118
119 with pytest.warns(DeprecationWarning, match="abcd"):
120 dummy(old_arg=1)
121 +
122 +
123 +def test_get_open_port():
124 + os.environ["VLLM_PORT"] = "5678"
125 + # make sure we can get multiple ports, even if the env var is set
126 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s1:
127 + s1.bind(("localhost", get_open_port()))
128 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s2:
129 + s2.bind(("localhost", get_open_port()))
130 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s3:
131 + s3.bind(("localhost", get_open_port()))
132 + os.environ.pop("VLLM_PORT")
vllm/envs.py
+3
@@ -99,6 +99,9 @@ environment_variables: Dict[str, Callable[[], Any]] = {
99 lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
100
101 # used in distributed environment to manually set the communication port
102 + # Note: if VLLM_PORT is set, and some code asks for multiple ports, the
103 + # VLLM_PORT will be used as the first port, and the rest will be generated
104 + # by incrementing the VLLM_PORT value.
105 # '0' is used to make mypy happy
106 'VLLM_PORT':
107 lambda: int(os.getenv('VLLM_PORT', '0'))
vllm/utils.py
+9 -1
@@ -289,7 +289,15 @@ def get_distributed_init_method(ip: str, port: int) -> str:
289 def get_open_port() -> int:
290 port = envs.VLLM_PORT
291 if port is not None:
292 - return port
292 + while True:
293 + try:
294 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
295 + s.bind(("", port))
296 + return port
297 + except OSError:
298 + port += 1 # Increment port number if already in use
299 + logger.info("Port %d is already in use, trying port %d",
300 + port - 1, port)
301 # try ipv4
302 try:
303 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: