A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 06:15 UTC
[BugFix] Partial revert of #29558 (DeepEP HT + PIECEWISE CG support) (#30910)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Lucas Wilkinson committed
Dec 18, 2025 at 02:50 UTC
30bb19a760d6d5e8c69b3a4c78c9cb7430872a61
2 files changed
+14
-74
tests/compile/test_config.py
-38
@@ -233,24 +233,6 @@ def test_splitting_ops_dynamic():
233
assert config.compilation_config.cudagraph_mode == CUDAGraphMode.PIECEWISE
234
235
236
-def test_moe_splitting_ops_deepep_ht_piecewise():
237
- # Non-inductor, non-attn-fusion case: DeepEP HT with dp>1
238
- # should add MoE ops to splitting_ops on top of attention ops.
239
- config = VllmConfig(
240
- parallel_config=ParallelConfig(
241
- all2all_backend="deepep_high_throughput",
242
- data_parallel_size=8,
243
- ),
244
- compilation_config=CompilationConfig(
245
- mode=CompilationMode.VLLM_COMPILE,
246
- ),
247
- )
248
- splitting_ops = config.compilation_config.splitting_ops
249
- assert splitting_ops is not None
250
- assert "vllm::moe_forward" in splitting_ops
251
- assert "vllm::moe_forward_shared" in splitting_ops
252
-
253
-
236
def test_moe_splitting_ops_deepep_ht_inductor_partition():
237
# Inductor partition case: user-provided splitting_ops should be
238
# preserved and MoE ops should be appended for DeepEP HT with dp>1.
@@ -277,26 +259,6 @@ def test_moe_splitting_ops_deepep_ht_inductor_partition():
259
]
260
261
280
-def test_moe_splitting_ops_deepep_ht_attn_fusion_no_inductor():
281
- # Pure attn-fusion case without inductor partition: even with
282
- # DeepEP HT and dp>1, we should not re-enable piecewise compilation
283
- # or add MoE ops into splitting_ops.
284
- config = VllmConfig(
285
- parallel_config=ParallelConfig(
286
- all2all_backend="deepep_high_throughput",
287
- data_parallel_size=8,
288
- ),
289
- compilation_config=CompilationConfig(
290
- mode=CompilationMode.VLLM_COMPILE,
291
- pass_config={"fuse_attn_quant": True, "eliminate_noops": True},
292
- custom_ops=["+quant_fp8"],
293
- cudagraph_mode=CUDAGraphMode.PIECEWISE,
294
- ),
295
- )
296
- assert config.compilation_config.splitting_ops == []
297
- assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL
298
-
299
-
262
def test_should_split():
263
import torch
264
vllm/config/compilation.py
+14
-36
@@ -915,8 +915,6 @@ class CompilationConfig:
915
"mode is CompilationMode.VLLM_COMPILE"
916
)
917
918
- added_default_splitting_ops = False
919
-
918
if self.pass_config.fuse_attn_quant and not self.use_inductor_graph_partition:
919
self.set_splitting_ops_for_attn_fusion()
920
else:
@@ -930,7 +928,6 @@ class CompilationConfig:
928
# for details. Make a copy to avoid mutating the class-level
929
# list via reference.
930
self.splitting_ops = list(self._attention_ops)
933
- added_default_splitting_ops = True
931
elif len(self.splitting_ops) == 0:
932
if (
933
self.cudagraph_mode == CUDAGraphMode.PIECEWISE
@@ -958,44 +955,25 @@ class CompilationConfig:
955
self.cudagraph_mode = CUDAGraphMode.FULL
956
self.splitting_ops = []
957
961
- # split MoE ops for cudagraph
962
- moe_ops = [
963
- "vllm::moe_forward",
964
- "vllm::moe_forward_shared",
965
- ]
958
+ # Disable CUDA graphs for DeepEP high-throughput since its not CG compatible
959
backend = all2all_backend or envs.VLLM_ALL2ALL_BACKEND
960
dp_size = data_parallel_size if data_parallel_size is not None else 1
968
- need_moe_splitting = (
961
+ if (
962
backend == "deepep_high_throughput"
963
and dp_size > 1
971
- # pure attn-fusion without inductor partition deliberately disables
972
- # piecewise graphs and MoE splitting.
973
- and not (
974
- self.pass_config.fuse_attn_quant
975
- and not self.use_inductor_graph_partition
964
+ and self.cudagraph_mode != CUDAGraphMode.NONE
965
+ ):
966
+ # TODO: Piecewise Cuda graph might be enabled
967
+ # if torch compile cache key issue fixed
968
+ # See https://github.com/vllm-project/vllm/pull/25093
969
+ logger.info(
970
+ "DeepEP: Disabling CUDA Graphs since DeepEP high-throughput kernels "
971
+ "are optimized for prefill and are incompatible with CUDA Graphs. "
972
+ "In order to use CUDA Graphs for decode-optimized workloads, "
973
+ "use --all2all-backend with another option, such as "
974
+ "deepep_low_latency, pplx, or allgather_reducescatter."
975
)
977
- )
978
-
979
- if need_moe_splitting and self.cudagraph_mode != CUDAGraphMode.NONE:
980
- # if we just initialized default splitting_ops for this config,
981
- # automatically append the MoE ops
982
- if added_default_splitting_ops:
983
- for op in moe_ops:
984
- if op not in self.splitting_ops:
985
- self.splitting_ops.append(op)
986
-
987
- # make sure MoE ops are split out
988
- if not any(op in self.splitting_ops for op in moe_ops):
989
- self.cudagraph_mode = CUDAGraphMode.NONE
990
- logger.warning_once(
991
- "DeepEP high throughput backend with data_parallel_size > 1 "
992
- "requires splitting MoE ops from cudagraphs. Please ensure "
993
- "'vllm::moe_forward' or 'vllm::moe_forward_shared' are "
994
- "present in CompilationConfig.splitting_ops."
995
- )
996
- elif self.cudagraph_mode.has_full_cudagraphs():
997
- # fall back to piecewise when MoE splitting is required.
998
- self.cudagraph_mode = CUDAGraphMode.PIECEWISE
976
+ self.cudagraph_mode = CUDAGraphMode.NONE
977
978
def set_splitting_ops_for_attn_fusion(self):
979
assert self.pass_config.fuse_attn_quant