A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Bugfix] Fix bnb 8bit model weights loading (#19917)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jun 21, 2025 at 11:29 UTC
6f170f11dddcfafa061785d4fb4993f7bcb16107
1 file changed
+2
-2
vllm/model_executor/model_loader/bitsandbytes_loader.py
+2
-2
@@ -577,10 +577,10 @@ def dequantize_dq(quant_states: dict) -> None:
577
thereby avoiding this computational overhead during inference. This comes
578
at the cost of increased memory usage.
579
"""
580
- from bitsandbytes.functional import dequantize_blockwise
580
+ from bitsandbytes.functional import QuantState, dequantize_blockwise
581
for _, quant_state in quant_states.items():
582
# Copied from: https://github.com/bitsandbytes-foundation/bitsandbytes/blob/0.45.3/bitsandbytes/functional.py#L1352-#L1356
583
- if quant_state.nested:
583
+ if isinstance(quant_state, QuantState) and quant_state.nested:
584
absmax = dequantize_blockwise(quant_state.absmax,
585
quant_state.state2)
586
absmax += quant_state.offset