@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b9c64c0ca7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Modify BNB parameter name (#9997)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 6, 2024 at 03:40 UTC b9c64c0ca79ccdea608f337fbb7e4b0c75fe3aac
3 files changed +11 -14
vllm/model_executor/layers/quantization/bitsandbytes.py
+5 -4
@@ -203,8 +203,9 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
203 qweight = create_qweight_for_8bit()
204 else:
205 qweight = create_qweight_for_4bit()
206 -
207 - layer.register_parameter("qweight", qweight)
206 + # Enable parameters to have the same name as in the BNB
207 + # checkpoint format.
208 + layer.register_parameter("weight", qweight)
209 set_weight_attrs(qweight, extra_weight_attrs)
210
211 def apply(self,
@@ -234,7 +235,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
235 reshape_after_matmul = True
236 bf_x = x.to(torch.bfloat16)
237
237 - qweight = layer.qweight
238 + qweight = layer.weight
239 offsets = qweight.bnb_shard_offsets
240 quant_states = qweight.bnb_quant_state
241 matmul_states = qweight.matmul_state
@@ -313,7 +314,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
314 reshape_after_matmul = True
315 bf_x = x.to(torch.bfloat16)
316
316 - qweight = layer.qweight
317 + qweight = layer.weight
318 quant_states = qweight.bnb_quant_state
319 offsets = qweight.bnb_shard_offsets
320
vllm/model_executor/layers/resampler.py
+1 -1
@@ -177,7 +177,7 @@ class BaseResampler(nn.Module):
177 embed_dim,
178 bias=False,
179 quant_config=quant_config,
180 - prefix=prefix)
180 + prefix=f"{prefix}.kv_proj")
181 else:
182 # Maintain the same return value with ReplicatedLinear.forward
183 self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
vllm/model_executor/model_loader/loader.py
+5 -9
@@ -892,7 +892,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
892 if not weight_name.lower().endswith(".scb"):
893 continue
894
895 - weight_key = weight_name.lower().replace(".scb", ".qweight")
895 + weight_key = weight_name.lower().replace(".scb", ".weight")
896 quant_state_dict[weight_key] = weight_tensor
897
898 for weight_name, weight_tensor in self._hf_weight_iter(
@@ -901,11 +901,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
901 if self._is_8bit_weight_name(weight_name):
902 continue
903
904 - qweight_name = weight_name.replace(".weight", ".qweight")
905 -
906 - if qweight_name in quant_state_dict:
904 + if weight_name in quant_state_dict:
905 set_weight_attrs(weight_tensor, {"load_in_8bit": True})
908 - yield qweight_name, weight_tensor
906 + yield weight_name, weight_tensor
907 else:
908 yield weight_name, weight_tensor
909
@@ -950,9 +948,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
948 (f"{weight_name}.quant_state.bitsandbytes__fp4" \
949 in temp_state_dict):
950 quant_state = _parse_quant_state(weight_name, temp_state_dict)
953 - weight_name = weight_name.replace(".weight", ".qweight")
951 quant_state_dict[weight_name] = quant_state
955 - yield weight_name.replace(".weight", ".qweight"), weight_tensor
952 + yield weight_name, weight_tensor
953 else:
954 yield weight_name, weight_tensor
955
@@ -967,7 +964,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
964
965 if any(target_module in weight_name for target_module in
966 self.target_modules) and weight_name.endswith(".weight"):
970 - weight_name = weight_name.replace(".weight", ".qweight")
967 # Without sharding
968 if any(
969 weight_name.startswith(module)
@@ -1093,7 +1089,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1089 # Some models, such as MiniCPM V2.5/2.6, contain both
1090 # module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
1091 # from being incorrectly identified as being present in
1096 - # 'vpm.encoder.layers.0.self_attn.qkv_proj.qweight
1092 + # 'vpm.encoder.layers.0.self_attn.qkv_proj.weight
1093 if shard_pos > 0 and quant_param_name[shard_pos - 1] == ".":
1094 shard_index = index
1095 quant_param_name = quant_param_name.replace(