A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Distributed] Basic set of configuration for large EP deployment on GB200 (#27328)
Signed-off-by: Pengchao Wang <wpc@fb.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
Pengchao Wang committed
Oct 24, 2025 at 14:16 UTC
d95d0f4b985f28ea381e301490f9d479b34d8980
2 files changed
+25
-1
vllm/distributed/device_communicators/all2all.py
+3
-1
@@ -277,7 +277,7 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
277
num_rdma_bytes = None
278
num_qps_per_rank = None
279
280
- if self.internode:
280
+ if self.internode and not envs.VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE:
281
num_rdma_bytes = envs.VLLM_DEEPEP_BUFFER_SIZE_MB * 1024 * 1024
282
num_qps_per_rank = self.num_sms // 2
283
else:
@@ -363,6 +363,8 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
363
num_rdma_bytes=num_rdma_bytes,
364
low_latency_mode=True,
365
num_qps_per_rank=num_qps_per_rank,
366
+ allow_nvlink_for_low_latency_mode=envs.VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK,
367
+ allow_mnnvl=envs.VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL,
368
)
369
370
def get_handle(self, kwargs):
vllm/envs.py
+22
@@ -204,6 +204,9 @@ if TYPE_CHECKING:
204
VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
205
VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME: str = "VLLM_OBJECT_STORAGE_SHM_BUFFER"
206
VLLM_DEEPEP_BUFFER_SIZE_MB: int = 1024
207
+ VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE: bool = False
208
+ VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK: bool = False
209
+ VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL: bool = False
210
VLLM_DBO_COMM_SMS: int = 20
211
GPT_OSS_SYSTEM_TOOL_MCP_LABELS: list[str] = []
212
VLLM_PATTERN_MATCH_DEBUG: str | None = None
@@ -1350,6 +1353,22 @@ environment_variables: dict[str, Callable[[], Any]] = {
1353
"VLLM_DEEPEP_BUFFER_SIZE_MB": lambda: int(
1354
os.getenv("VLLM_DEEPEP_BUFFER_SIZE_MB", "1024")
1355
),
1356
+ # Force DeepEP to use intranode kernel for inter-node communication in
1357
+ # high throughput mode. This is useful archive higher prefill throuhgput
1358
+ # on system supports multi-node nvlink (e.g GB200).
1359
+ "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE": lambda: bool(
1360
+ int(os.getenv("VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", "0"))
1361
+ ),
1362
+ # Allow DeepEP to use nvlink for internode_ll kernel, turn this on for
1363
+ # better latency on GB200 like system
1364
+ "VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK": lambda: bool(
1365
+ int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK", "0"))
1366
+ ),
1367
+ # Allow DeepEP to use MNNVL (multi-node nvlink) for internode_ll kernel,
1368
+ # turn this for better latency on GB200 like system
1369
+ "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool(
1370
+ int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0"))
1371
+ ),
1372
# The number of SMs to allocate for communication kernels when running DBO
1373
# the rest of the SMs on the device will be allocated to compute
1374
"VLLM_DBO_COMM_SMS": lambda: int(os.getenv("VLLM_DBO_COMM_SMS", "20")),
@@ -1512,6 +1531,9 @@ def compute_hash() -> str:
1531
"VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING",
1532
"VLLM_NVFP4_GEMM_BACKEND",
1533
"VLLM_USE_FBGEMM",
1534
+ "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE",
1535
+ "VLLM_DEEPEP_LOW_LATENCY_ALLOW_NVLINK",
1536
+ "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL",
1537
]
1538
for key in environment_variables_to_hash:
1539
# if this goes out of sync with environment_variables,