@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 37a4947dcd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix layer skip logic with bitsandbytes (#9887)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 1, 2024 at 01:12 UTC 37a4947dcd68c602d0911920e2c1a9168dea1ecb
1 file changed +6 -1
vllm/model_executor/layers/quantization/bitsandbytes.py
+6 -1
@@ -119,7 +119,12 @@ class BitsAndBytesConfig(QuantizationConfig):
119
120
121 def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
122 - return any(module_name in prefix for module_name in llm_int8_skip_modules)
122 + # Split the prefix into its dot-separated components
123 + components = prefix.split('.')
124 +
125 + # Check if any of the skip modules exactly matches any component
126 + return any(module_name in components
127 + for module_name in llm_int8_skip_modules)
128
129
130 class BitsAndBytesLinearMethod(LinearMethodBase):