@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c5e8330997
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix null `modules_to_not_convert` in FBGEMM Fp8 quantization (#6665)

Cheng Li committed Jul 22, 2024 at 19:25 UTC c5e8330997dc3969818c6696a79820bcee44a702
1 file changed +1 -1
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+1 -1
@@ -31,7 +31,7 @@ class FBGEMMFp8Config(QuantizationConfig):
31 """Config class for FBGEMM Fp8."""
32
33 def __init__(self, ignore_list: List[str], input_scale_ub: float):
34 - self.ignore_list = ignore_list
34 + self.ignore_list = ignore_list if ignore_list else []
35 self.input_scale_ub = input_scale_ub
36
37 # For GPUs that lack FP8 hardware support, we can leverage the Marlin