@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cfb3bf25fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[bugfix] fix the default value of llm_int8_threshold in BitsAndBytesConfig (#10657)

yansh97 committed Nov 27, 2024 at 13:55 UTC cfb3bf25fb981494fa6c575fb0714388c9df99b0
1 file changed +2 -2
vllm/model_executor/layers/quantization/bitsandbytes.py
+2 -2
@@ -26,7 +26,7 @@ class BitsAndBytesConfig(QuantizationConfig):
26 llm_int8_enable_fp32_cpu_offload: bool = False,
27 llm_int8_has_fp16_weight: bool = False,
28 llm_int8_skip_modules: Optional[List[str]] = None,
29 - llm_int8_threshold: float = 0.0,
29 + llm_int8_threshold: float = 6.0,
30 ) -> None:
31
32 self.load_in_8bit = load_in_8bit
@@ -103,7 +103,7 @@ class BitsAndBytesConfig(QuantizationConfig):
103 ["llm_int8_skip_modules"],
104 default_value=[])
105 llm_int8_threshold = get_safe_value(config, ["llm_int8_threshold"],
106 - default_value=0.0)
106 + default_value=6.0)
107
108 return cls(
109 load_in_8bit=load_in_8bit,