A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Allow Unsloth Dynamic 4bit BnB quants to work (#12974)
Daniel Han committed
Feb 12, 2025 at 23:13 UTC
cb944d5818398281d5e777824ee394d5455d8b8b
1 file changed
+10
-2
vllm/model_executor/layers/quantization/bitsandbytes.py
+10
-2
@@ -133,8 +133,16 @@ def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
133
components = prefix.split('.')
134
135
# Check if any of the skip modules exactly matches any component
136
- return any(module_name in components
137
- for module_name in llm_int8_skip_modules)
136
+ substr_check = any(module_name in components
137
+ for module_name in llm_int8_skip_modules)
138
+
139
+ # Allow certain layers to not be quantized
140
+ set_components = set(".".join(components[:i + 1])
141
+ for i in range(len(components)))
142
+ set_llm_int8_skip_modules = set(llm_int8_skip_modules)
143
+ prefix_check = len(set_llm_int8_skip_modules & set_components) != 0
144
+
145
+ return substr_check or prefix_check
146
147
148
class BitsAndBytesLinearMethod(LinearMethodBase):