@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cb944d5818
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Allow Unsloth Dynamic 4bit BnB quants to work (#12974)

Daniel Han committed Feb 12, 2025 at 23:13 UTC cb944d5818398281d5e777824ee394d5455d8b8b
1 file changed +10 -2
vllm/model_executor/layers/quantization/bitsandbytes.py
+10 -2
@@ -133,8 +133,16 @@ def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
133 components = prefix.split('.')
134
135 # Check if any of the skip modules exactly matches any component
136 - return any(module_name in components
137 - for module_name in llm_int8_skip_modules)
136 + substr_check = any(module_name in components
137 + for module_name in llm_int8_skip_modules)
138 +
139 + # Allow certain layers to not be quantized
140 + set_components = set(".".join(components[:i + 1])
141 + for i in range(len(components)))
142 + set_llm_int8_skip_modules = set(llm_int8_skip_modules)
143 + prefix_check = len(set_llm_int8_skip_modules & set_components) != 0
144 +
145 + return substr_check or prefix_check
146
147
148 class BitsAndBytesLinearMethod(LinearMethodBase):