A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 21:15 UTC
[ROCm] Guard group quant RMS norm fusion patterns (#30239)
Ye (Charlotte) Qi committed
Dec 8, 2025 at 06:44 UTC
eb1051fb95323493d6d950c03dabac8ee56cb33e
1 file changed
+19
-17
vllm/compilation/fusion.py
+19
-17
@@ -490,23 +490,25 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass):
490
# as the latter is a subset of the former in torch ops
491
for epsilon in [1e-5, 1e-6]:
492
# Fuse fused_add_rms_norm + fp8 group quant
493
- FusedAddRMSNormGroupQuantPattern(
494
- epsilon, FP8_DTYPE, group_shape=GroupShape(1, 128)
495
- ).register(self.patterns)
496
-
497
- # Fuse rms_norm + fp8 group quant
498
- RMSNormGroupQuantPattern(
499
- epsilon, FP8_DTYPE, group_shape=GroupShape(1, 128)
500
- ).register(self.patterns)
501
-
502
- FusedAddRMSNormGroupQuantPattern(
503
- epsilon, FP8_DTYPE, group_shape=GroupShape(1, 64)
504
- ).register(self.patterns)
505
-
506
- # Fuse rms_norm + fp8 group quant
507
- RMSNormGroupQuantPattern(
508
- epsilon, FP8_DTYPE, group_shape=GroupShape(1, 64)
509
- ).register(self.patterns)
493
+ # Only register group quant patterns on CUDA where the C++ op exists
494
+ if current_platform.is_cuda():
495
+ FusedAddRMSNormGroupQuantPattern(
496
+ epsilon, FP8_DTYPE, group_shape=GroupShape(1, 128)
497
+ ).register(self.patterns)
498
+
499
+ # Fuse rms_norm + fp8 group quant
500
+ RMSNormGroupQuantPattern(
501
+ epsilon, FP8_DTYPE, group_shape=GroupShape(1, 128)
502
+ ).register(self.patterns)
503
+
504
+ FusedAddRMSNormGroupQuantPattern(
505
+ epsilon, FP8_DTYPE, group_shape=GroupShape(1, 64)
506
+ ).register(self.patterns)
507
+
508
+ # Fuse rms_norm + fp8 group quant
509
+ RMSNormGroupQuantPattern(
510
+ epsilon, FP8_DTYPE, group_shape=GroupShape(1, 64)
511
+ ).register(self.patterns)
512
513
# Fuse fused_add_rms_norm + static fp8 quant
514
FusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(