A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc] Improve BNB loader to handle mixture of sharded and merged weights with same suffix (#11566)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 28, 2024 at 03:45 UTC
dde1fa18c9f9ba992a8300a300543d6c18d5f08d
1 file changed
+5
-2
vllm/model_executor/model_loader/loader.py
+5
-2
@@ -1001,8 +1001,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1001
for sub_name in sub_modules:
1002
self.target_modules.append(
1003
name.replace(last_name, sub_name))
1004
- else:
1005
- self.target_modules.append(name)
1004
+ # Add original module name even if the module has stacked map,
1005
+ # in case model has a mixture of disk-merged and disk-splitted
1006
+ # weights with same last name.
1007
+ self.target_modules.append(name)
1008
+
1009
assert (self.target_modules
1010
), "vllm currently does not support BNB quantization for"
1011
f" {type(model).__name__}"