A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Bug Fix] Fix address/port already in use error for deep_ep test (#20094)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Jun 26, 2025 at 10:33 UTC
c894c5dc1ffadee8979f3a051bfccea0441ae09a
2 files changed
+13
-1
tests/kernels/moe/deepep_utils.py
+4
-1
@@ -4,6 +4,7 @@ DeepEP test utilities
4
"""
5
import dataclasses
6
import importlib
7
+import os
8
import traceback
9
from typing import Callable, Optional
10
@@ -13,6 +14,8 @@ from torch.multiprocessing import (
14
spawn) # pyright: ignore[reportPrivateImportUsage]
15
from typing_extensions import Concatenate, ParamSpec
16
17
+from vllm.model_executor.layers.fused_moe.utils import find_free_port
18
+
19
has_deep_ep = importlib.util.find_spec("deep_ep") is not None
20
if has_deep_ep:
21
from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import ( # noqa: E501
@@ -92,7 +95,7 @@ def parallel_launch(
95
world_size,
96
world_size,
97
0,
95
- "tcp://localhost:29500",
98
+ f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{find_free_port()}",
99
worker,
100
) + args,
101
nprocs=world_size,
vllm/model_executor/layers/fused_moe/utils.py
+9
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import socket
4
+from contextlib import closing
5
from math import prod
6
from typing import Optional
7
@@ -96,3 +98,10 @@ def _fp8_perm(m: torch.Tensor, idx: torch.Tensor) -> torch.Tensor:
98
return m.view(dtype=torch.uint8)[idx, ...].view(dtype=m.dtype)
99
else:
100
return m[idx, ...]
101
+
102
+
103
+def find_free_port():
104
+ with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
105
+ s.bind(('', 0))
106
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
107
+ return s.getsockname()[1]
\ No newline at end of file