A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc] Add BNB quantization for Whisper (#12381)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Feb 4, 2025 at 16:27 UTC
96b23621c16d4e3b65380c6af3a7d7bac79cfa5b
3 files changed
+82
-44
vllm/model_executor/model_loader/loader.py
+60
-42
@@ -803,9 +803,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
803
iterator = safetensors_weights_iterator(hf_weights_files)
804
else:
805
iterator = pt_weights_iterator(hf_weights_files)
806
- for name, param in iterator:
807
- # mapping weight names from transformers to vllm.
808
- yield self.weight_mapper(name), param
806
+ for org_name, param in iterator:
807
+ # mapping weight names from transformers to vllm while preserving
808
+ # original names.
809
+ mapped_name = self.weight_mapper(org_name)
810
+ yield org_name, mapped_name, param
811
812
def _get_quantized_weights_iterator(
813
self,
@@ -866,24 +868,30 @@ class BitsAndBytesModelLoader(BaseModelLoader):
868
869
def _quantized_8bit_generator(self, hf_weights_files, use_safetensors,
870
quant_state_dict) -> Generator:
869
- for weight_name, weight_tensor in self._hf_weight_iter(
870
- hf_weights_files, use_safetensors):
871
- if not weight_name.lower().endswith(".scb"):
871
+ for (
872
+ org_weight_name,
873
+ mapped_weight_name,
874
+ weight_tensor,
875
+ ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
876
+ if not mapped_weight_name.lower().endswith(".scb"):
877
continue
878
874
- weight_key = weight_name.lower().replace(".scb", ".weight")
879
+ weight_key = mapped_weight_name.lower().replace(".scb", ".weight")
880
quant_state_dict[weight_key] = weight_tensor
881
877
- for weight_name, weight_tensor in self._hf_weight_iter(
878
- hf_weights_files, use_safetensors):
879
- if self._is_8bit_weight_name(weight_name):
882
+ for (
883
+ org_weight_name,
884
+ mapped_weight_name,
885
+ weight_tensor,
886
+ ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
887
+ if self._is_8bit_weight_name(mapped_weight_name):
888
continue
889
882
- if weight_name in quant_state_dict:
890
+ if mapped_weight_name in quant_state_dict:
891
set_weight_attrs(weight_tensor, {"load_in_8bit": True})
884
- yield weight_name, weight_tensor
892
+ yield org_weight_name, weight_tensor
893
else:
886
- yield weight_name, weight_tensor
894
+ yield org_weight_name, weight_tensor
895
896
def _quantized_4bit_generator(self, hf_weights_files, use_safetensors,
897
quant_state_dict) -> Generator:
@@ -893,15 +901,19 @@ class BitsAndBytesModelLoader(BaseModelLoader):
901
weight_iterator = self._hf_weight_iter(hf_weights_files,
902
use_safetensors)
903
temp_state_dict = {}
896
- for weight_name, weight_tensor in weight_iterator:
897
- if not self._is_4bit_weight_name(weight_name):
904
+ for (
905
+ org_weight_name,
906
+ mapped_weight_name,
907
+ weight_tensor,
908
+ ) in weight_iterator:
909
+ if not self._is_4bit_weight_name(mapped_weight_name):
910
continue
911
# bitsandbytes library requires
912
# weight.quant_state.bitsandbytes__* in CPU
901
- if "quant_state.bitsandbytes" in weight_name:
902
- temp_state_dict[weight_name] = weight_tensor.cpu().data
913
+ if "quant_state.bitsandbytes" in mapped_weight_name:
914
+ temp_state_dict[mapped_weight_name] = weight_tensor.cpu().data
915
else:
904
- temp_state_dict[weight_name] = weight_tensor
916
+ temp_state_dict[mapped_weight_name] = weight_tensor
917
918
# Closure to parse quant_state for each prequant weight
919
def _parse_quant_state(param_name: str,
@@ -915,20 +927,24 @@ class BitsAndBytesModelLoader(BaseModelLoader):
927
928
# Second iterate over all prequant and normal weights
929
# pre quantized weights would have a quant_state
918
- for weight_name, weight_tensor in self._hf_weight_iter(
919
- hf_weights_files, use_safetensors):
920
- if self._is_4bit_weight_name(weight_name):
930
+ for (
931
+ org_weight_name,
932
+ mapped_weight_name,
933
+ weight_tensor,
934
+ ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
935
+ if self._is_4bit_weight_name(mapped_weight_name):
936
continue
937
923
- if (f"{weight_name}.quant_state.bitsandbytes__nf4"
938
+ if (f"{mapped_weight_name}.quant_state.bitsandbytes__nf4"
939
in temp_state_dict) or (
925
- f"{weight_name}.quant_state.bitsandbytes__fp4"
940
+ f"{mapped_weight_name}.quant_state.bitsandbytes__fp4"
941
in temp_state_dict):
927
- quant_state = _parse_quant_state(weight_name, temp_state_dict)
928
- quant_state_dict[weight_name] = quant_state
929
- yield weight_name, weight_tensor
942
+ quant_state = _parse_quant_state(mapped_weight_name,
943
+ temp_state_dict)
944
+ quant_state_dict[mapped_weight_name] = quant_state
945
+ yield org_weight_name, weight_tensor
946
else:
931
- yield weight_name, weight_tensor
947
+ yield org_weight_name, weight_tensor
948
949
def _unquantized_generator(self, hf_weights_files, use_safetensors,
950
quant_state_dict) -> Generator:
@@ -937,18 +953,22 @@ class BitsAndBytesModelLoader(BaseModelLoader):
953
tp_size = get_tensor_model_parallel_world_size()
954
tp_rank = get_tensor_model_parallel_rank()
955
940
- for weight_name, weight_tensor in self._hf_weight_iter(
941
- hf_weights_files, use_safetensors):
942
- if any(target_module in weight_name for target_module in
943
- self.target_modules) and weight_name.endswith(".weight"):
956
+ for (
957
+ org_weight_name,
958
+ mapped_weight_name,
959
+ weight_tensor,
960
+ ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
961
+ if any(target_module in mapped_weight_name
962
+ for target_module in self.target_modules
963
+ ) and mapped_weight_name.endswith(".weight"):
964
# Without sharding
965
if any(
946
- weight_name.startswith(module)
966
+ mapped_weight_name.startswith(module)
967
for module in self.unsharded_weights_modules):
968
weight_sub_tensor = weight_tensor
969
# Shard by column
970
elif any(
951
- weight_name.startswith(module)
971
+ mapped_weight_name.startswith(module)
972
for module in self.column_sharded_weights_modules):
973
total_size = weight_tensor.size(-1)
974
start_index = total_size // tp_size * tp_rank
@@ -958,14 +978,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
978
# Weights have fused on disk. In this case, we assume that the
979
# weight and module use same name.
980
elif any(
961
- weight_name.startswith(module)
981
+ mapped_weight_name.startswith(module)
982
for module in self.maybe_fused_weights_modules):
983
# special case for fused weights
984
# get the size of each shard weight tensor
985
total_shard_sizes = next(
986
(sizes for module, sizes in
987
self.maybe_fused_weights_modules.items()
968
- if weight_name.startswith(module)))
988
+ if mapped_weight_name.startswith(module)))
989
total_size = weight_tensor.size(0)
990
assert total_size == sum(total_shard_sizes)
991
# get the start/end index of each shard weight tensor
@@ -1008,23 +1028,21 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1028
quant_type="nf4",
1029
)
1030
1011
- quant_state_dict[weight_name] = quant_state
1031
+ quant_state_dict[mapped_weight_name] = quant_state
1032
else:
1033
processed_weight = weight_tensor
1014
-
1015
- yield weight_name, processed_weight
1034
+ yield org_weight_name, processed_weight
1035
1036
def _get_bnb_target_modules(self, model: nn.Module) -> None:
1037
1038
for name, module in model.named_modules():
1039
if isinstance(module, (LinearBase, )):
1021
- last_name = name.split(".")[-1]
1022
- if sub_modules := self.modules_mapping.packed_mapping.get(
1023
- last_name, []):
1040
+ if modules_info := self.modules_mapping.get_sub_modules(name):
1041
# Map vllm's names to transformers's names.
1042
+ rep_name, sub_modules = modules_info
1043
for sub_name in sub_modules:
1044
self.target_modules.append(
1027
- name.replace(last_name, sub_name))
1045
+ name.replace(rep_name, sub_name))
1046
# Add original module name even if the module has stacked map,
1047
# in case model has a mixture of disk-merged and disk-splitted
1048
# weights with same last name.
vllm/model_executor/model_loader/utils.py
+7
@@ -131,3 +131,10 @@ class ParamMapping:
131
packed_name,
132
index,
133
)
134
+
135
+ def get_sub_modules(self,
136
+ module_name: str) -> Optional[Tuple[str, List[str]]]:
137
+ for key, value in self.packed_mapping.items():
138
+ if module_name.endswith(key):
139
+ return key, value
140
+ return None
vllm/model_executor/models/whisper.py
+15
-2
@@ -638,6 +638,19 @@ def input_mapper_for_whisper(
638
@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
639
"audio", get_max_whisper_audio_tokens)
640
class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
641
+ packed_modules_mapping = {
642
+ "self_attn.qkv_proj": [
643
+ "self_attn.q_proj",
644
+ "self_attn.k_proj",
645
+ "self_attn.v_proj",
646
+ ],
647
+ "encoder_attn.kv_proj": ["encoder_attn.k_proj", "encoder_attn.v_proj"],
648
+ }
649
+
650
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={
651
+ ".fc1.": ".mlp.fc1.",
652
+ ".fc2.": ".mlp.fc2."
653
+ })
654
655
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
656
super().__init__()
@@ -731,10 +744,10 @@ class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
744
def load_weights(self, weights: Iterable[Tuple[str,
745
torch.Tensor]]) -> Set[str]:
746
loader = AutoWeightsLoader(self, skip_prefixes=["proj_out."])
734
- mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
747
+
748
# add fake zeros bias for k_proj to state_dict
749
weights = _create_fake_bias_for_k_proj(weights)
737
- return loader.load_weights(weights, mapper=mapper)
750
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
751
752
753
def _create_fake_bias_for_k_proj(