A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 16:15 UTC
[ROCm][CI] Fix TestSiluMulGroupFp8QuantModel after W8A8 block linear refactor (#39799)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Apr 24, 2026 at 21:20 UTC
e54894fc85a9861fb38a49701b5844462c3d77e4
1 file changed
+19
-10
tests/compile/passes/test_silu_mul_quant_fusion.py
+19
-10
@@ -10,7 +10,7 @@ import vllm.envs as envs
10
from tests.compile.backend import TestBackend
11
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
12
from tests.utils import TestFP8Layer
13
-from vllm._aiter_ops import IS_AITER_FOUND
13
+from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
14
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
15
from vllm.compilation.passes.fusion.act_quant_fusion import (
16
FUSED_OPS,
@@ -157,12 +157,13 @@ class TestSiluMulGroupFp8QuantModel(torch.nn.Module):
157
activation_quant_key=self.act_quant_key,
158
input_dtype=dtype,
159
)
160
- self.w = torch.rand(hidden_size, hidden_size).to(dtype=FP8_DTYPE).t()
160
162
- scale_hidden_size = (hidden_size + 128 - 1) // 128
163
- self.wscale = torch.rand(
164
- (scale_hidden_size, scale_hidden_size), dtype=torch.float32
165
- )
161
+ if not current_platform.is_fp8_fnuz():
162
+ kernel = self.w8a8_block_fp8_linear.kernel
163
+ orig_quant = kernel.quant_fp8
164
+ kernel.quant_fp8 = lambda *a, use_triton=False, **kw: orig_quant(
165
+ *a, use_triton=True, **kw
166
+ )
167
168
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
169
@@ -174,6 +175,9 @@ class TestSiluMulGroupFp8QuantModel(torch.nn.Module):
175
def ops_in_model_before(self):
176
return [
177
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
178
+ rocm_aiter_ops.get_group_quant_op()
179
+ if current_platform.is_fp8_fnuz()
180
+ else torch.ops.vllm.triton_per_token_group_quant_fp8.default,
181
]
182
183
def ops_in_model_after(self):
@@ -324,7 +328,6 @@ def test_fusion_silu_and_mul_quant(
328
with set_current_vllm_config(config), monkeypatch.context() as m:
329
fusion_passes = [ActivationQuantFusionPass(config)]
330
if IS_AITER_FOUND and model_class is TestSiluMulGroupFp8QuantModel:
327
- from vllm._aiter_ops import rocm_aiter_ops
331
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
332
RocmAiterSiluMulFp8GroupQuantFusionPass,
333
)
@@ -352,10 +355,16 @@ def test_fusion_silu_and_mul_quant(
355
atol, rtol = 1e-3, 1e-3
356
elif isinstance(model, TestSiluMulNvfp4QuantModel):
357
atol, rtol = 1e-1, 1e-1
355
- elif isinstance(
356
- model, (TestSiluMulGroupFp8QuantModel, TestSiluMulBlockQuantModel)
357
- ):
358
+ elif isinstance(model, TestSiluMulGroupFp8QuantModel):
359
atol, rtol = 5e-2, 5e-2
360
+ elif isinstance(model, TestSiluMulBlockQuantModel):
361
+ if current_platform.is_rocm():
362
+ atol, rtol = 1e-3, 1e-3
363
+ else:
364
+ # CUDA fused kernel computes silu*mul in fp32 while the reference
365
+ # goes through bf16/fp16 storage, so group maxima (and thus scales)
366
+ # can shift by one FP8-e4m3 code (~1/8 relative step).
367
+ atol, rtol = 5e-2, 5e-2
368
369
torch.testing.assert_close(
370
result[0].to(dtype=dtype), result2[0].to(dtype=dtype), atol=atol, rtol=rtol