A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fix dynamic FP8 quantization for Mixtral (#4793)
Philipp Moritz committed
May 13, 2024 at 16:00 UTC
33d3914b1e6d85a855da1a69193030c1915cb6f9
1 file changed
+1
-1
vllm/model_executor/models/mixtral.py
+1
-1
@@ -95,7 +95,7 @@ class MixtralMoE(nn.Module):
95
params_dtype=self.params_dtype,
96
quant_config=None)
97
98
- if self.use_fp8:
98
+ if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
99
params_dtype = torch.float8_e4m3fn
100
101
self.w13_weight = nn.Parameter(