A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] Fix torch Compilation Cache Hit Error (#25093)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Sep 18, 2025 at 15:38 UTC
d2a30a2d933226d3951ad98cb5de0c74e2e64826
2 files changed
+10
-19
vllm/config/compilation.py
-12
@@ -563,18 +563,6 @@ class CompilationConfig:
563
self.cudagraph_mode = CUDAGraphMode.FULL
564
self.splitting_ops = []
565
566
- if envs.VLLM_ALL2ALL_BACKEND == "deepep_high_throughput":
567
- # exclude MoE dispatch/combine from capture by ensuring
568
- # piecewise splitting includes them, so communication remains
569
- # outside CUDA graphs while compute can still be graphed.
570
- moe_ops = [
571
- "vllm.moe_forward",
572
- "vllm.moe_forward_shared",
573
- ]
574
- for op in moe_ops:
575
- if op not in self.splitting_ops:
576
- self.splitting_ops.append(op)
577
-
566
def splitting_ops_contain_attention(self) -> bool:
567
return self.splitting_ops is not None and all(
568
op in self.splitting_ops for op in self._attention_ops)
vllm/platforms/cuda.py
+10
-7
@@ -191,14 +191,17 @@ class CudaPlatformBase(Platform):
191
compilation_config = vllm_config.compilation_config
192
if (envs.VLLM_ALL2ALL_BACKEND == "deepep_high_throughput"
193
and parallel_config.data_parallel_size > 1
194
- and compilation_config.cudagraph_mode
195
- not in [CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE]):
194
+ and compilation_config.cudagraph_mode != CUDAGraphMode.NONE):
195
+ # TODO: Piecewise Cuda graph might be enabled
196
+ # if torch compile cache key issue fixed
197
+ # See https://github.com/vllm-project/vllm/pull/25093
198
logger.info(
197
- "Data Parallel with DeepEP high-throughput: using PIECEWISE "
198
- "CUDA graphs and excluding MoE ops from capture. Set "
199
- "VLLM_ALL2ALL_BACKEND=deepep_low_latency if you need MoE "
200
- "graphs captured as well.")
201
- compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
199
+ "Data Parallel: disabling cudagraphs since DP "
200
+ "with DeepEP high-throughput kernels are not CUDA Graph "
201
+ "compatible. The DeepEP low-latency kernels are CUDA Graph "
202
+ "compatible. Set the all_to_all backend to deepep_low_latency "
203
+ "to use those kernels instead.")
204
+ compilation_config.cudagraph_mode = CUDAGraphMode.NONE
205
206
@classmethod
207
def get_current_memory_usage(cls,