@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 33d3914b1e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fix dynamic FP8 quantization for Mixtral (#4793)

Philipp Moritz committed May 13, 2024 at 16:00 UTC 33d3914b1e6d85a855da1a69193030c1915cb6f9
1 file changed +1 -1
vllm/model_executor/models/mixtral.py
+1 -1
@@ -95,7 +95,7 @@ class MixtralMoE(nn.Module):
95 params_dtype=self.params_dtype,
96 quant_config=None)
97
98 - if self.use_fp8:
98 + if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
99 params_dtype = torch.float8_e4m3fn
100
101 self.w13_weight = nn.Parameter(