A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Misc] Slight improvement of the BNB (#19418)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Jee Jee Li committed
Jun 10, 2025 at 21:51 UTC
b6553be1bc75f046b00046a4ad7576364d03c835
2 files changed
+3
-4
vllm/model_executor/layers/quantization/bitsandbytes.py
+1
-3
@@ -71,9 +71,7 @@ class BitsAndBytesConfig(QuantizationConfig):
71
72
@staticmethod
73
def get_config_filenames() -> list[str]:
74
- return [
75
- "adapter_config.json",
76
- ]
74
+ return []
75
76
@classmethod
77
def from_config(cls, config: dict[str, Any]) -> "BitsAndBytesConfig":
vllm/model_executor/model_loader/bitsandbytes_loader.py
+2
-1
@@ -392,7 +392,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
392
def _get_bnb_target_modules(self, model: nn.Module) -> None:
393
394
for name, module in model.named_modules():
395
- if isinstance(module, (LinearBase, )):
395
+ if (isinstance(module, LinearBase) and
396
+ hasattr(module.quant_method, "quant_config")):
397
if modules_info := self.modules_mapping.get_sub_modules(name):
398
# Map vllm's names to transformers's names.
399
rep_name, sub_modules = modules_info