A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[ROCm][Bugfix][FP8] Make fp8 quant respect fused modules mapping (#16031)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Apr 7, 2025 at 19:28 UTC
21802c4b6d96ba1b6a85913c01a3784b55a12248
1 file changed
+3
-1
vllm/model_executor/layers/quantization/fp8.py
+3
-1
@@ -116,7 +116,9 @@ class Fp8Config(QuantizationConfig):
116
from vllm.attention.layer import Attention # Avoid circular import
117
118
if isinstance(layer, LinearBase):
119
- if is_layer_skipped(prefix, self.ignored_layers):
119
+ if is_layer_skipped(prefix=prefix,
120
+ ignored_layers=self.ignored_layers,
121
+ fused_mapping=self.packed_modules_mapping):
122
return UnquantizedLinearMethod()
123
return Fp8LinearMethod(self)
124
elif isinstance(layer, FusedMoE):