@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d7d95a70a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Offload port selection to OS (#467)

Zhangir Azerbayev committed Jul 16, 2023 at 02:11 UTC 6d7d95a70a4308699ad60fb5bcc08904135086f2
1 file changed +9 -3
vllm/engine/ray_utils.py
+9 -3
@@ -1,4 +1,4 @@
1 -import random
1 +import socket
2 from typing import List, Optional, Tuple
3
4 try:
@@ -12,6 +12,12 @@ from vllm.config import ParallelConfig
12 DeviceID = Tuple[int, Optional[str], int]
13
14
15 +def get_open_port():
16 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
17 + s.bind(("", 0))
18 + return s.getsockname()[1]
19 +
20 +
21 def initialize_cluster(
22 parallel_config: ParallelConfig,
23 engine_use_ray: bool = False,
@@ -42,7 +48,7 @@ def initialize_cluster(
48
49 if not parallel_config.worker_use_ray:
50 # Initialize cluster locally.
45 - port = random.randint(10000, 20000)
51 + port = get_open_port()
52 # We need to setup the distributed init method to make sure
53 # the distributed megatron code (e.g., get world size) works correctly.
54 distributed_init_method = f"tcp://localhost:{port}"
@@ -96,7 +102,7 @@ def initialize_cluster(
102 stage_devices.append((rank, node_resource, current_device_id))
103 if distributed_init_method is None:
104 ip = node_resource.split("node:")[-1]
99 - port = random.randint(10000, 20000)
105 + port = get_open_port()
106 distributed_init_method = f"tcp://{ip}:{port}"
107 rank += 1
108 current_device_id += 1