A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
Remove all2all backend envvar (#30363)
Signed-off-by: Elizabeth Thomas <email2eliza@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Elizabeth Thomas committed
Dec 18, 2025 at 13:46 UTC
41b6f9200fef27cd43b3299408a6f0e50654931f
12 files changed
+40
-43
.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh
+1
-1
@@ -44,10 +44,10 @@ trap cleanup EXIT
44
45
for BACK in "${BACKENDS[@]}"; do
46
VLLM_DEEP_GEMM_WARMUP=skip \
47
- VLLM_ALL2ALL_BACKEND=$BACK \
47
vllm serve "$MODEL" \
48
--enforce-eager \
49
--enable-eplb \
50
+ --all2all-backend $BACK \
51
--eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true}' \
52
--tensor-parallel-size ${TENSOR_PARALLEL_SIZE} \
53
--data-parallel-size ${DATA_PARALLEL_SIZE} \
.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh
+1
-1
@@ -43,12 +43,12 @@ trap cleanup EXIT
43
44
for BACK in "${BACKENDS[@]}"; do
45
VLLM_DEEP_GEMM_WARMUP=skip \
46
- VLLM_ALL2ALL_BACKEND=$BACK \
46
vllm serve "$MODEL" \
47
--enforce-eager \
48
--tensor-parallel-size 4 \
49
--enable-expert-parallel \
50
--enable-eplb \
51
+ --all2all-backend $BACK \
52
--eplb-config '{"window_size":200,"step_interval":600,"use_async":true}' \
53
--speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}' \
54
--trust-remote-code \
.buildkite/test-amd.yaml
+1
-1
@@ -1497,7 +1497,7 @@ steps:
1497
- "VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/distributed/test_fusions_e2e.py -k 'not Llama-4'"
1498
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/distributed/test_sequence_parallel.py
1499
- pytest -v -s tests/distributed/test_context_parallel.py
1500
- - HIP_VISIBLE_DEVICES=0,1 VLLM_ALL2ALL_BACKEND=deepep_high_throughput VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048
1500
+ - HIP_VISIBLE_DEVICES=0,1 VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048 --all2all-backend deepep_high_throughput
1501
- pytest -v -s tests/v1/distributed/test_dbo.py
1502
1503
##### B200 test #####
.buildkite/test-pipeline.yaml
+1
-1
@@ -1331,7 +1331,7 @@ steps:
1331
- "VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/distributed/test_fusions_e2e.py -k 'not Llama-4'"
1332
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/distributed/test_sequence_parallel.py
1333
- pytest -v -s tests/distributed/test_context_parallel.py
1334
- - CUDA_VISIBLE_DEVICES=1,2 VLLM_ALL2ALL_BACKEND=deepep_high_throughput VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048
1334
+ - CUDA_VISIBLE_DEVICES=1,2 VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048 --all2all-backend deepep_high_throughput
1335
- pytest -v -s tests/v1/distributed/test_dbo.py
1336
1337
##### B200 test #####
.buildkite/test_areas/distributed.yaml
+1
-1
@@ -145,7 +145,7 @@ steps:
145
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/distributed/test_fusions_e2e.py -k 'not Llama-4'
146
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/distributed/test_sequence_parallel.py
147
- pytest -v -s tests/distributed/test_context_parallel.py
148
- - CUDA_VISIBLE_DEVICES=1,2 VLLM_ALL2ALL_BACKEND=deepep_high_throughput VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048
148
+ - CUDA_VISIBLE_DEVICES=1,2 VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048 --all2all-backend deepep_high_throughput
149
- pytest -v -s tests/v1/distributed/test_dbo.py
150
151
- label: Distributed Tests (2 GPUs)(B200)
docs/design/moe_kernel_features.md
+1
-1
@@ -16,7 +16,7 @@ Async backends support the use of DBO (Dual Batch Overlap) and shared expert ove
16
17
Certain models require the topk weights to be applied to the input activations rather than the output activations when topk==1, e.g. Llama. For modular kernels, this feature is supported by the `FusedMoEPrepareAndFinalize` subclass. For non-modular kernels, it is up to the experts function to deal with this flag.
18
19
-Unless otherwise specified, backends are controlled via `VLLM_ALL2ALL_BACKEND`. All backends except `flashinfer` only work with EP+DP or EP+TP. `Flashinfer` can work with EP or DP without EP.
19
+Unless otherwise specified, backends are controlled via the `--all2all-backend` command-line argument (or the `all2all_backend` parameter in `ParallelConfig`). All backends except `flashinfer` only work with EP+DP or EP+TP. `Flashinfer` can work with EP or DP without EP.
20
21
<style>
22
td {
examples/online_serving/elastic_ep/serve_deepseek_v2.sh
+1
-1
@@ -55,7 +55,6 @@ done
55
echo "Starting vLLM server for $MODEL_NAME with data parallel size: $DATA_PARALLEL_SIZE and redundant experts: $REDUNDANT_EXPERTS"
56
57
export RAY_DEDUP_LOGS=0
58
-export VLLM_ALL2ALL_BACKEND="pplx"
58
export VLLM_USE_DEEP_GEMM=1
59
60
vllm serve $MODEL_NAME \
@@ -65,6 +64,7 @@ vllm serve $MODEL_NAME \
64
--enforce-eager \
65
--enable-expert-parallel \
66
--enable-eplb \
67
+ --all2all-backend pplx \
68
--num-redundant-experts $REDUNDANT_EXPERTS \
69
--trust-remote-code \
70
--host $HOST \
tests/v1/cudagraph/test_cudagraph_dispatch.py
+4
-1
@@ -49,7 +49,10 @@ def _create_vllm_config(
49
mock_config.lora_config = None
50
# Mimic the behavior of VllmConfig.__post_init__()
51
if compilation_config.mode == CompilationMode.VLLM_COMPILE:
52
- compilation_config.set_splitting_ops_for_v1()
52
+ compilation_config.set_splitting_ops_for_v1(
53
+ all2all_backend=mock_config.parallel_config.all2all_backend,
54
+ data_parallel_size=mock_config.parallel_config.data_parallel_size,
55
+ )
56
57
# mimic VllmConfig.__post_init__
58
if compilation_config.cudagraph_capture_sizes:
vllm/config/compilation.py
+3
-5
@@ -899,7 +899,7 @@ class CompilationConfig:
899
self.compute_bs_to_padded_graph_size()
900
901
def set_splitting_ops_for_v1(
902
- self, all2all_backend: str | None = None, data_parallel_size: int | None = None
902
+ self, all2all_backend: str, data_parallel_size: int = 1
903
):
904
# To compatible with OOT hardware plugin platform (for example vllm-ascend)
905
# which currently only supports sequence parallelism in eager mode.
@@ -956,11 +956,9 @@ class CompilationConfig:
956
self.splitting_ops = []
957
958
# Disable CUDA graphs for DeepEP high-throughput since its not CG compatible
959
- backend = all2all_backend or envs.VLLM_ALL2ALL_BACKEND
960
- dp_size = data_parallel_size if data_parallel_size is not None else 1
959
if (
962
- backend == "deepep_high_throughput"
963
- and dp_size > 1
960
+ all2all_backend == "deepep_high_throughput"
961
+ and data_parallel_size > 1
962
and self.cudagraph_mode != CUDAGraphMode.NONE
963
):
964
# TODO: Piecewise Cuda graph might be enabled
vllm/config/parallel.py
+22
-27
@@ -36,6 +36,14 @@ ExpertPlacementStrategy = Literal["linear", "round_robin"]
36
DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"]
37
DataParallelBackend = Literal["ray", "mp"]
38
EPLBPolicyOption = Literal["default"]
39
+All2AllBackend = Literal[
40
+ "naive",
41
+ "pplx",
42
+ "deepep_high_throughput",
43
+ "deepep_low_latency",
44
+ "allgather_reducescatter",
45
+ "flashinfer_all2allv",
46
+]
47
48
49
@config
@@ -126,24 +134,14 @@ class ParallelConfig:
134
with 4 experts and 2 ranks, rank 0 will have experts [0, 2] and rank 1
135
will have experts [1, 3]. This strategy can help improve load balancing
136
for grouped expert models with no redundant experts."""
129
- all2all_backend: (
130
- Literal[
131
- "naive",
132
- "pplx",
133
- "deepep_high_throughput",
134
- "deepep_low_latency",
135
- "allgather_reducescatter",
136
- "flashinfer_all2allv",
137
- ]
138
- | None
139
- ) = None
140
- """All2All backend for MoE expert parallel communication. If not set, uses
141
- the value from VLLM_ALL2ALL_BACKEND environment variable. Available options:
142
- - "naive": Naive all2all implementation using broadcasts
143
- - "allgather_reducescatter": All2all based on allgather and reducescatter
144
- - "pplx": Use pplx kernels
145
- - "deepep_high_throughput": Use deepep high-throughput kernels
146
- - "deepep_low_latency": Use deepep low-latency kernels
137
+ all2all_backend: All2AllBackend = "allgather_reducescatter"
138
+ """All2All backend for MoE expert parallel communication. Available options:
139
+
140
+ - "naive": Naive all2all implementation using broadcasts\n
141
+ - "allgather_reducescatter": All2all based on allgather and reducescatter\n
142
+ - "pplx": Use pplx kernels\n
143
+ - "deepep_high_throughput": Use deepep high-throughput kernels\n
144
+ - "deepep_low_latency": Use deepep low-latency kernels\n
145
- "flashinfer_all2allv": Use flashinfer alltoallv kernels for mnnvl"""
146
147
max_parallel_loading_workers: int | None = None
@@ -495,20 +493,17 @@ class ParallelConfig:
493
from vllm.config.utils import get_hash_factors, hash_factors
494
495
factors = get_hash_factors(self, ignored_factors)
498
- # Explicitly include backend affecting env factor as before
499
- factors["VLLM_ALL2ALL_BACKEND"] = str(envs.VLLM_ALL2ALL_BACKEND)
496
return hash_factors(factors)
497
498
def __post_init__(self) -> None:
499
# Set all2all_backend from env var if not specified, with deprecation warning
504
- if self.all2all_backend is None:
500
+ if envs.is_set("VLLM_ALL2ALL_BACKEND"):
501
+ logger.warning_once(
502
+ "VLLM_ALL2ALL_BACKEND environment variable is deprecated and "
503
+ "will be removed in v0.15.0. Please use the "
504
+ "--all2all-backend command-line argument instead."
505
+ )
506
self.all2all_backend = envs.VLLM_ALL2ALL_BACKEND
506
- if envs.is_set("VLLM_ALL2ALL_BACKEND"):
507
- logger.warning_once(
508
- "VLLM_ALL2ALL_BACKEND environment variable is deprecated and "
509
- "will be removed in a future release. Please use the "
510
- "--all2all-backend command-line argument instead."
511
- )
507
508
# Continue with the rest of the initialization
509
self.world_size = (
vllm/engine/arg_utils.py
+1
-1
@@ -408,7 +408,7 @@ class EngineArgs:
408
data_parallel_external_lb: bool = False
409
data_parallel_backend: str = ParallelConfig.data_parallel_backend
410
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
411
- all2all_backend: str | None = ParallelConfig.all2all_backend
411
+ all2all_backend: str = ParallelConfig.all2all_backend
412
enable_dbo: bool = ParallelConfig.enable_dbo
413
ubatch_size: int = ParallelConfig.ubatch_size
414
dbo_decode_token_threshold: int = ParallelConfig.dbo_decode_token_threshold
vllm/envs.py
+3
-2
@@ -1263,7 +1263,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
1263
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
1264
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
1265
),
1266
- # all2all backend for vllm's expert parallel communication
1266
+ # [DEPRECATED - will be removed in v0.15.0] all2all backend for vllm's
1267
+ # expert parallel communication. Use --all2all-backend CLI argument instead.
1268
# Available options:
1269
# - "naive": naive all2all implementation using broadcasts
1270
# - "allgather_reducescatter": all2all implementation based on allgather and
@@ -1274,7 +1275,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
1275
# - "flashinfer_all2allv", use flashinfer alltoallv kernels for mnnvl
1276
"VLLM_ALL2ALL_BACKEND": env_with_choices(
1277
"VLLM_ALL2ALL_BACKEND",
1277
- "allgather_reducescatter",
1278
+ None,
1279
[
1280
"naive",
1281
"pplx",