A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix port conflict by obtaining a list of open ports upfront (#21894)
Signed-off-by: Ming Yang <minos.future@gmail.com>
Ming Yang committed
Aug 21, 2025 at 10:22 UTC
10f535c0863ffcd23ae5838979ea4e0dd82c5886
3 files changed
+35
-13
vllm/config/parallel.py
+18
-6
@@ -15,7 +15,7 @@ import vllm.envs as envs
15
from vllm.config.utils import config
16
from vllm.logger import init_logger
17
from vllm.platforms import current_platform
18
-from vllm.utils import cuda_device_count_stateless, get_open_port
18
+from vllm.utils import cuda_device_count_stateless, get_open_ports_list
19
20
if TYPE_CHECKING:
21
from ray.runtime_env import RuntimeEnv
@@ -171,6 +171,11 @@ class ParallelConfig:
171
rank: int = 0
172
"""Global rank in distributed setup."""
173
174
+ _data_parallel_master_port_list: list[int] = field(default_factory=list)
175
+ """List of open port auto-queried for data parallel messaging.
176
+ Set to be private as it's not intended to be configured by users.
177
+ """
178
+
179
@property
180
def world_size_across_dp(self) -> int:
181
"""world_size_across_dp is TPxPPxDP, it is the size of the world
@@ -183,11 +188,15 @@ class ParallelConfig:
188
processes that is related to data parallelism,
189
e.g. both in the worker and in the engine, which
190
can live in different processes. To avoid port conflicts, we
186
- increment the port number each time we need to initialize a
187
- new process group related to data parallelism.
191
+ pop a new port from the prepared port list each time we need to
192
+ initialize a new process group related to data parallelism.
193
"""
189
- answer = self.data_parallel_master_port
190
- self.data_parallel_master_port += 1
194
+ if self._data_parallel_master_port_list:
195
+ answer = self._data_parallel_master_port_list.pop()
196
+ else:
197
+ answer = self.data_parallel_master_port
198
+ self.data_parallel_master_port += 1
199
+
200
return answer
201
202
def stateless_init_dp_group(self) -> ProcessGroup:
@@ -313,7 +322,10 @@ class ParallelConfig:
322
323
if self.data_parallel_size > 1 or self.data_parallel_size_local == 0:
324
# Data parallel was specified in the engine args.
316
- self.data_parallel_master_port = get_open_port()
325
+ if not self._data_parallel_master_port_list:
326
+ self._data_parallel_master_port_list = get_open_ports_list(5)
327
+ self.data_parallel_master_port = \
328
+ self._data_parallel_master_port_list.pop()
329
330
if not (0 <= self.data_parallel_rank < self.data_parallel_size):
331
raise ValueError(
vllm/utils/__init__.py
+14
-6
@@ -516,8 +516,8 @@ def random_uuid() -> str:
516
class AsyncMicrobatchTokenizer:
517
"""Asynchronous tokenizer with micro-batching.
518
519
- Pulls pending encode/decode requests from a queue and batches them
520
- up to reduce overhead. A single-thread ThreadPoolExecutor is used
519
+ Pulls pending encode/decode requests from a queue and batches them
520
+ up to reduce overhead. A single-thread ThreadPoolExecutor is used
521
so the event loop stays responsive.
522
"""
523
@@ -664,18 +664,18 @@ class AsyncMicrobatchTokenizer:
664
def _queue_key(self, op: str, kwargs: dict) -> tuple:
665
"""
666
Return a normalized key describing operation + kwargs.
667
-
667
+
668
- `add_special_tokens`: {True/False}
669
- `truncation`: {True/False}
670
- - If `truncation` is False (`max_length` is None),
670
+ - If `truncation` is False (`max_length` is None),
671
returns a key for a can_batch queue.
672
- If `truncation` is True and `max_length` is None or equals
673
`tokenizer.model_max_length`, returns a key for a can_batch queue.
674
- Otherwise, returns a key for a cannot_batch queue.
675
-
675
+
676
Examples:
677
- Decode: ("decode",)
678
- - Encode typical:
678
+ - Encode typical:
679
("encode", add_special_tokens, bool_truncation, max_length_label)
680
- Fallback: ("encode", "other")
681
"""
@@ -940,6 +940,14 @@ def get_open_port() -> int:
940
return _get_open_port()
941
942
943
+def get_open_ports_list(count: int = 5) -> list[int]:
944
+ """Get a list of open ports."""
945
+ ports = set()
946
+ while len(ports) < count:
947
+ ports.add(get_open_port())
948
+ return list(ports)
949
+
950
+
951
def _get_open_port() -> int:
952
port = envs.VLLM_PORT
953
if port is not None:
vllm/v1/engine/utils.py
+3
-1
@@ -71,7 +71,7 @@ class EngineHandshakeMetadata:
71
connect to.
72
"""
73
addresses: EngineZmqAddresses
74
- parallel_config: dict[str, Union[int, str]]
74
+ parallel_config: dict[str, Union[int, str, list[int]]]
75
76
77
class CoreEngineProcManager:
@@ -798,6 +798,8 @@ def wait_for_engine_startup(
798
parallel_config.data_parallel_master_ip,
799
"data_parallel_master_port":
800
parallel_config.data_parallel_master_port,
801
+ "_data_parallel_master_port_list":
802
+ parallel_config._data_parallel_master_port_list,
803
"data_parallel_size":
804
parallel_config.data_parallel_size,
805
}))