A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[torch.compile] Use custom ops when use_inductor=False (#19618)
Woosuk Kwon committed
Jun 13, 2025 at 15:05 UTC
aafbbd981f7ce9871af1df8e156df27791f8e602
1 file changed
+15
-3
vllm/config.py
+15
-3
@@ -4450,15 +4450,27 @@ class VllmConfig:
4450
self.compilation_config.custom_ops.append("+rms_norm")
4451
if envs.VLLM_USE_V1 and self.model_config is not None and \
4452
not self.model_config.enforce_eager:
4453
- # FIXME(rob): Add function to set all of these.
4454
- if not self.compilation_config.custom_ops:
4455
- self.compilation_config.custom_ops = ["none"]
4453
+ # By default, V1 uses piecewise CUDA graphs. If full_cuda_graph
4454
+ # is set to True, full CUDA graphs will be used.
4455
self.compilation_config.cudagraph_num_of_warmups = 1
4456
self.compilation_config.pass_config.enable_fusion = False
4457
self.compilation_config.pass_config.enable_noop = False
4458
self.compilation_config.level = CompilationLevel.PIECEWISE
4459
self.compilation_config.set_splitting_ops_for_v1()
4460
4461
+ # The behavior of custom ops with inductor depends on the config:
4462
+ # - If use_inductor=True and custom_ops is empty:
4463
+ # Inductor generates Triton kernels for all registered custom ops
4464
+ # (default behavior)
4465
+ # - If use_inductor=True and custom_ops is non-empty:
4466
+ # Custom CUDA kernels are used for specified ops while inductor
4467
+ # generates Triton kernels for remaining ops, including misc torch
4468
+ # ops in the model.
4469
+ if (not self.compilation_config.custom_ops
4470
+ and self.compilation_config.use_inductor):
4471
+ # Let inductor generate Triton kernels for the custom ops.
4472
+ self.compilation_config.custom_ops = ["none"]
4473
+
4474
self._set_cudagraph_sizes()
4475
4476
if self.cache_config.cpu_offload_gb > 0 and \