A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Offload port selection to OS (#467)
Zhangir Azerbayev committed
Jul 16, 2023 at 02:11 UTC
6d7d95a70a4308699ad60fb5bcc08904135086f2
1 file changed
+9
-3
vllm/engine/ray_utils.py
+9
-3
@@ -1,4 +1,4 @@
1
-import random
1
+import socket
2
from typing import List, Optional, Tuple
3
4
try:
@@ -12,6 +12,12 @@ from vllm.config import ParallelConfig
12
DeviceID = Tuple[int, Optional[str], int]
13
14
15
+def get_open_port():
16
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
17
+ s.bind(("", 0))
18
+ return s.getsockname()[1]
19
+
20
+
21
def initialize_cluster(
22
parallel_config: ParallelConfig,
23
engine_use_ray: bool = False,
@@ -42,7 +48,7 @@ def initialize_cluster(
48
49
if not parallel_config.worker_use_ray:
50
# Initialize cluster locally.
45
- port = random.randint(10000, 20000)
51
+ port = get_open_port()
52
# We need to setup the distributed init method to make sure
53
# the distributed megatron code (e.g., get world size) works correctly.
54
distributed_init_method = f"tcp://localhost:{port}"
@@ -96,7 +102,7 @@ def initialize_cluster(
102
stage_devices.append((rank, node_resource, current_device_id))
103
if distributed_init_method is None:
104
ip = node_resource.split("node:")[-1]
99
- port = random.randint(10000, 20000)
105
+ port = get_open_port()
106
distributed_init_method = f"tcp://{ip}:{port}"
107
rank += 1
108
current_device_id += 1