A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
use combo kernel to fuse qk-norm and qk-rope (#26682)
Signed-off-by: Boyuan Feng <boyuan@meta.com>
Boyuan Feng committed
Oct 14, 2025 at 06:40 UTC
ca683a2a729d894286e7fef6afcb4d34b75e37ca
1 file changed
+10
vllm/config/compilation.py
+10
@@ -513,6 +513,16 @@ class CompilationConfig:
513
if isinstance(self.pass_config, dict):
514
self.pass_config = PassConfig(**self.pass_config)
515
516
+ if (
517
+ is_torch_equal_or_newer("2.9.0.dev")
518
+ and "combo_kernels" not in self.inductor_compile_config
519
+ and "benchmark_combo_kernel" not in self.inductor_compile_config
520
+ ):
521
+ # use horizontal fusion, which is useful for fusing qk-norm and
522
+ # qk-rope when query and key have different shapes.
523
+ self.inductor_compile_config["combo_kernels"] = True
524
+ self.inductor_compile_config["benchmark_combo_kernel"] = True
525
+
526
# migrate the deprecated flags
527
if not self.use_cudagraph:
528
logger.warning(