A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 12:30 UTC
[BugFix] Fix async EPLB hang with DeepEP LL all2all backend (#32860)
Signed-off-by: ilmarkov <markovilya197@gmail.com>
Ilya Markov committed
Feb 10, 2026 at 23:34 UTC
bb2fc8b5e7beca9c5749e464b4607c753db0b630
2 files changed
+56
vllm/distributed/eplb/eplb_utils.py
new
+54
@@ -0,0 +1,54 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Utility functions for EPLB (Expert Parallel Load Balancing)."""
4
+
5
+import os
6
+
7
+from vllm.config import ParallelConfig
8
+from vllm.logger import init_logger
9
+
10
+logger = init_logger(__name__)
11
+
12
+
13
+def override_envs_for_eplb(parallel_config: ParallelConfig) -> None:
14
+ """
15
+ Override environment variables for EPLB when specific conditions are met.
16
+
17
+ Args:
18
+ parallel_config: The parallel configuration object.
19
+ """
20
+ is_data_parallel = parallel_config.data_parallel_size > 1
21
+ is_eplb_enabled = parallel_config.enable_eplb
22
+ async_eplb = parallel_config.eplb_config.use_async
23
+ is_deepep_ll = parallel_config.all2all_backend == "deepep_low_latency"
24
+
25
+ # Override NCCL_MAX_CTAS to avoid hangs when using async EPLB with the
26
+ # DeepEP low-latency backend.
27
+ #
28
+ # The hang happens when two ranks interleave kernel launches differently
29
+ # between NCCL collectives (used by async EPLB weight exchange) and DeepEP
30
+ # low-latency (LL) kernels. DeepEP LL uses a cooperative launch and tries
31
+ # to reserve a large fraction of the GPU's SMs; if those SMs are currently
32
+ # occupied by NCCL, the DeepEP LL launch blocks until enough SMs are
33
+ # freed.
34
+ #
35
+ # If rank A enters DeepEP LL in main thread while rank B is still executing
36
+ # NCCL in async thread, rank A can block waiting for SMs, while rank B can
37
+ # block inside NCCL waiting for rank A to participate in the collective.
38
+ # This circular wait causes a deadlock.
39
+ # Limiting NCCL occupancy via NCCL_MAX_CTAS leaves space for the DeepEP
40
+ # cooperative kernel to launch and complete, breaking the deadlock.
41
+ # See: https://github.com/deepseek-ai/DeepEP/issues/496
42
+ if is_data_parallel and is_eplb_enabled and is_deepep_ll and async_eplb:
43
+ current_value_str = os.getenv("NCCL_MAX_CTAS")
44
+
45
+ if current_value_str and current_value_str.isdigit():
46
+ return
47
+
48
+ override_value = 8
49
+ os.environ["NCCL_MAX_CTAS"] = str(override_value)
50
+ logger.info_once(
51
+ f"EPLB: Setting NCCL_MAX_CTAS={override_value} "
52
+ "for expert parallel with EPLB and deepep_low_latency backend",
53
+ scope="global",
54
+ )
vllm/v1/worker/gpu_worker.py
+2
@@ -22,6 +22,7 @@ from vllm.distributed import (
22
set_custom_all_reduce,
23
)
24
from vllm.distributed.ec_transfer import ensure_ec_transfer_initialized
25
+from vllm.distributed.eplb.eplb_utils import override_envs_for_eplb
26
from vllm.distributed.kv_transfer import (
27
ensure_kv_transfer_initialized,
28
ensure_kv_transfer_shutdown,
@@ -1035,6 +1036,7 @@ def init_worker_distributed_environment(
1036
from vllm.model_executor.layers.batch_invariant import init_batch_invariance
1037
1038
init_batch_invariance(attention_config.backend)
1039
+ override_envs_for_eplb(parallel_config)
1040
set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)
1041
1042
init_method = distributed_init_method or "env://"