A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix layer skip logic with bitsandbytes (#9887)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 1, 2024 at 01:12 UTC
37a4947dcd68c602d0911920e2c1a9168dea1ecb
1 file changed
+6
-1
vllm/model_executor/layers/quantization/bitsandbytes.py
+6
-1
@@ -119,7 +119,12 @@ class BitsAndBytesConfig(QuantizationConfig):
119
120
121
def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
122
- return any(module_name in prefix for module_name in llm_int8_skip_modules)
122
+ # Split the prefix into its dot-separated components
123
+ components = prefix.split('.')
124
+
125
+ # Check if any of the skip modules exactly matches any component
126
+ return any(module_name in components
127
+ for module_name in llm_int8_skip_modules)
128
129
130
class BitsAndBytesLinearMethod(LinearMethodBase):