A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Modify BNB parameter name (#9997)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 6, 2024 at 03:40 UTC
b9c64c0ca79ccdea608f337fbb7e4b0c75fe3aac
3 files changed
+11
-14
vllm/model_executor/layers/quantization/bitsandbytes.py
+5
-4
@@ -203,8 +203,9 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
203
qweight = create_qweight_for_8bit()
204
else:
205
qweight = create_qweight_for_4bit()
206
-
207
- layer.register_parameter("qweight", qweight)
206
+ # Enable parameters to have the same name as in the BNB
207
+ # checkpoint format.
208
+ layer.register_parameter("weight", qweight)
209
set_weight_attrs(qweight, extra_weight_attrs)
210
211
def apply(self,
@@ -234,7 +235,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
235
reshape_after_matmul = True
236
bf_x = x.to(torch.bfloat16)
237
237
- qweight = layer.qweight
238
+ qweight = layer.weight
239
offsets = qweight.bnb_shard_offsets
240
quant_states = qweight.bnb_quant_state
241
matmul_states = qweight.matmul_state
@@ -313,7 +314,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
314
reshape_after_matmul = True
315
bf_x = x.to(torch.bfloat16)
316
316
- qweight = layer.qweight
317
+ qweight = layer.weight
318
quant_states = qweight.bnb_quant_state
319
offsets = qweight.bnb_shard_offsets
320
vllm/model_executor/layers/resampler.py
+1
-1
@@ -177,7 +177,7 @@ class BaseResampler(nn.Module):
177
embed_dim,
178
bias=False,
179
quant_config=quant_config,
180
- prefix=prefix)
180
+ prefix=f"{prefix}.kv_proj")
181
else:
182
# Maintain the same return value with ReplicatedLinear.forward
183
self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
vllm/model_executor/model_loader/loader.py
+5
-9
@@ -892,7 +892,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
892
if not weight_name.lower().endswith(".scb"):
893
continue
894
895
- weight_key = weight_name.lower().replace(".scb", ".qweight")
895
+ weight_key = weight_name.lower().replace(".scb", ".weight")
896
quant_state_dict[weight_key] = weight_tensor
897
898
for weight_name, weight_tensor in self._hf_weight_iter(
@@ -901,11 +901,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
901
if self._is_8bit_weight_name(weight_name):
902
continue
903
904
- qweight_name = weight_name.replace(".weight", ".qweight")
905
-
906
- if qweight_name in quant_state_dict:
904
+ if weight_name in quant_state_dict:
905
set_weight_attrs(weight_tensor, {"load_in_8bit": True})
908
- yield qweight_name, weight_tensor
906
+ yield weight_name, weight_tensor
907
else:
908
yield weight_name, weight_tensor
909
@@ -950,9 +948,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
948
(f"{weight_name}.quant_state.bitsandbytes__fp4" \
949
in temp_state_dict):
950
quant_state = _parse_quant_state(weight_name, temp_state_dict)
953
- weight_name = weight_name.replace(".weight", ".qweight")
951
quant_state_dict[weight_name] = quant_state
955
- yield weight_name.replace(".weight", ".qweight"), weight_tensor
952
+ yield weight_name, weight_tensor
953
else:
954
yield weight_name, weight_tensor
955
@@ -967,7 +964,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
964
965
if any(target_module in weight_name for target_module in
966
self.target_modules) and weight_name.endswith(".weight"):
970
- weight_name = weight_name.replace(".weight", ".qweight")
967
# Without sharding
968
if any(
969
weight_name.startswith(module)
@@ -1093,7 +1089,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1089
# Some models, such as MiniCPM V2.5/2.6, contain both
1090
# module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
1091
# from being incorrectly identified as being present in
1096
- # 'vpm.encoder.layers.0.self_attn.qkv_proj.qweight
1092
+ # 'vpm.encoder.layers.0.self_attn.qkv_proj.weight
1093
if shard_pos > 0 and quant_param_name[shard_pos - 1] == ".":
1094
shard_index = index
1095
quant_param_name = quant_param_name.replace(