@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96b23621c1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc] Add BNB quantization for Whisper (#12381)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Feb 4, 2025 at 16:27 UTC 96b23621c16d4e3b65380c6af3a7d7bac79cfa5b
3 files changed +82 -44
vllm/model_executor/model_loader/loader.py
+60 -42
@@ -803,9 +803,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
803 iterator = safetensors_weights_iterator(hf_weights_files)
804 else:
805 iterator = pt_weights_iterator(hf_weights_files)
806 - for name, param in iterator:
807 - # mapping weight names from transformers to vllm.
808 - yield self.weight_mapper(name), param
806 + for org_name, param in iterator:
807 + # mapping weight names from transformers to vllm while preserving
808 + # original names.
809 + mapped_name = self.weight_mapper(org_name)
810 + yield org_name, mapped_name, param
811
812 def _get_quantized_weights_iterator(
813 self,
@@ -866,24 +868,30 @@ class BitsAndBytesModelLoader(BaseModelLoader):
868
869 def _quantized_8bit_generator(self, hf_weights_files, use_safetensors,
870 quant_state_dict) -> Generator:
869 - for weight_name, weight_tensor in self._hf_weight_iter(
870 - hf_weights_files, use_safetensors):
871 - if not weight_name.lower().endswith(".scb"):
871 + for (
872 + org_weight_name,
873 + mapped_weight_name,
874 + weight_tensor,
875 + ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
876 + if not mapped_weight_name.lower().endswith(".scb"):
877 continue
878
874 - weight_key = weight_name.lower().replace(".scb", ".weight")
879 + weight_key = mapped_weight_name.lower().replace(".scb", ".weight")
880 quant_state_dict[weight_key] = weight_tensor
881
877 - for weight_name, weight_tensor in self._hf_weight_iter(
878 - hf_weights_files, use_safetensors):
879 - if self._is_8bit_weight_name(weight_name):
882 + for (
883 + org_weight_name,
884 + mapped_weight_name,
885 + weight_tensor,
886 + ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
887 + if self._is_8bit_weight_name(mapped_weight_name):
888 continue
889
882 - if weight_name in quant_state_dict:
890 + if mapped_weight_name in quant_state_dict:
891 set_weight_attrs(weight_tensor, {"load_in_8bit": True})
884 - yield weight_name, weight_tensor
892 + yield org_weight_name, weight_tensor
893 else:
886 - yield weight_name, weight_tensor
894 + yield org_weight_name, weight_tensor
895
896 def _quantized_4bit_generator(self, hf_weights_files, use_safetensors,
897 quant_state_dict) -> Generator:
@@ -893,15 +901,19 @@ class BitsAndBytesModelLoader(BaseModelLoader):
901 weight_iterator = self._hf_weight_iter(hf_weights_files,
902 use_safetensors)
903 temp_state_dict = {}
896 - for weight_name, weight_tensor in weight_iterator:
897 - if not self._is_4bit_weight_name(weight_name):
904 + for (
905 + org_weight_name,
906 + mapped_weight_name,
907 + weight_tensor,
908 + ) in weight_iterator:
909 + if not self._is_4bit_weight_name(mapped_weight_name):
910 continue
911 # bitsandbytes library requires
912 # weight.quant_state.bitsandbytes__* in CPU
901 - if "quant_state.bitsandbytes" in weight_name:
902 - temp_state_dict[weight_name] = weight_tensor.cpu().data
913 + if "quant_state.bitsandbytes" in mapped_weight_name:
914 + temp_state_dict[mapped_weight_name] = weight_tensor.cpu().data
915 else:
904 - temp_state_dict[weight_name] = weight_tensor
916 + temp_state_dict[mapped_weight_name] = weight_tensor
917
918 # Closure to parse quant_state for each prequant weight
919 def _parse_quant_state(param_name: str,
@@ -915,20 +927,24 @@ class BitsAndBytesModelLoader(BaseModelLoader):
927
928 # Second iterate over all prequant and normal weights
929 # pre quantized weights would have a quant_state
918 - for weight_name, weight_tensor in self._hf_weight_iter(
919 - hf_weights_files, use_safetensors):
920 - if self._is_4bit_weight_name(weight_name):
930 + for (
931 + org_weight_name,
932 + mapped_weight_name,
933 + weight_tensor,
934 + ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
935 + if self._is_4bit_weight_name(mapped_weight_name):
936 continue
937
923 - if (f"{weight_name}.quant_state.bitsandbytes__nf4"
938 + if (f"{mapped_weight_name}.quant_state.bitsandbytes__nf4"
939 in temp_state_dict) or (
925 - f"{weight_name}.quant_state.bitsandbytes__fp4"
940 + f"{mapped_weight_name}.quant_state.bitsandbytes__fp4"
941 in temp_state_dict):
927 - quant_state = _parse_quant_state(weight_name, temp_state_dict)
928 - quant_state_dict[weight_name] = quant_state
929 - yield weight_name, weight_tensor
942 + quant_state = _parse_quant_state(mapped_weight_name,
943 + temp_state_dict)
944 + quant_state_dict[mapped_weight_name] = quant_state
945 + yield org_weight_name, weight_tensor
946 else:
931 - yield weight_name, weight_tensor
947 + yield org_weight_name, weight_tensor
948
949 def _unquantized_generator(self, hf_weights_files, use_safetensors,
950 quant_state_dict) -> Generator:
@@ -937,18 +953,22 @@ class BitsAndBytesModelLoader(BaseModelLoader):
953 tp_size = get_tensor_model_parallel_world_size()
954 tp_rank = get_tensor_model_parallel_rank()
955
940 - for weight_name, weight_tensor in self._hf_weight_iter(
941 - hf_weights_files, use_safetensors):
942 - if any(target_module in weight_name for target_module in
943 - self.target_modules) and weight_name.endswith(".weight"):
956 + for (
957 + org_weight_name,
958 + mapped_weight_name,
959 + weight_tensor,
960 + ) in self._hf_weight_iter(hf_weights_files, use_safetensors):
961 + if any(target_module in mapped_weight_name
962 + for target_module in self.target_modules
963 + ) and mapped_weight_name.endswith(".weight"):
964 # Without sharding
965 if any(
946 - weight_name.startswith(module)
966 + mapped_weight_name.startswith(module)
967 for module in self.unsharded_weights_modules):
968 weight_sub_tensor = weight_tensor
969 # Shard by column
970 elif any(
951 - weight_name.startswith(module)
971 + mapped_weight_name.startswith(module)
972 for module in self.column_sharded_weights_modules):
973 total_size = weight_tensor.size(-1)
974 start_index = total_size // tp_size * tp_rank
@@ -958,14 +978,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
978 # Weights have fused on disk. In this case, we assume that the
979 # weight and module use same name.
980 elif any(
961 - weight_name.startswith(module)
981 + mapped_weight_name.startswith(module)
982 for module in self.maybe_fused_weights_modules):
983 # special case for fused weights
984 # get the size of each shard weight tensor
985 total_shard_sizes = next(
986 (sizes for module, sizes in
987 self.maybe_fused_weights_modules.items()
968 - if weight_name.startswith(module)))
988 + if mapped_weight_name.startswith(module)))
989 total_size = weight_tensor.size(0)
990 assert total_size == sum(total_shard_sizes)
991 # get the start/end index of each shard weight tensor
@@ -1008,23 +1028,21 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1028 quant_type="nf4",
1029 )
1030
1011 - quant_state_dict[weight_name] = quant_state
1031 + quant_state_dict[mapped_weight_name] = quant_state
1032 else:
1033 processed_weight = weight_tensor
1014 -
1015 - yield weight_name, processed_weight
1034 + yield org_weight_name, processed_weight
1035
1036 def _get_bnb_target_modules(self, model: nn.Module) -> None:
1037
1038 for name, module in model.named_modules():
1039 if isinstance(module, (LinearBase, )):
1021 - last_name = name.split(".")[-1]
1022 - if sub_modules := self.modules_mapping.packed_mapping.get(
1023 - last_name, []):
1040 + if modules_info := self.modules_mapping.get_sub_modules(name):
1041 # Map vllm's names to transformers's names.
1042 + rep_name, sub_modules = modules_info
1043 for sub_name in sub_modules:
1044 self.target_modules.append(
1027 - name.replace(last_name, sub_name))
1045 + name.replace(rep_name, sub_name))
1046 # Add original module name even if the module has stacked map,
1047 # in case model has a mixture of disk-merged and disk-splitted
1048 # weights with same last name.
vllm/model_executor/model_loader/utils.py
+7
@@ -131,3 +131,10 @@ class ParamMapping:
131 packed_name,
132 index,
133 )
134 +
135 + def get_sub_modules(self,
136 + module_name: str) -> Optional[Tuple[str, List[str]]]:
137 + for key, value in self.packed_mapping.items():
138 + if module_name.endswith(key):
139 + return key, value
140 + return None
vllm/model_executor/models/whisper.py
+15 -2
@@ -638,6 +638,19 @@ def input_mapper_for_whisper(
638 @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
639 "audio", get_max_whisper_audio_tokens)
640 class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
641 + packed_modules_mapping = {
642 + "self_attn.qkv_proj": [
643 + "self_attn.q_proj",
644 + "self_attn.k_proj",
645 + "self_attn.v_proj",
646 + ],
647 + "encoder_attn.kv_proj": ["encoder_attn.k_proj", "encoder_attn.v_proj"],
648 + }
649 +
650 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={
651 + ".fc1.": ".mlp.fc1.",
652 + ".fc2.": ".mlp.fc2."
653 + })
654
655 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
656 super().__init__()
@@ -731,10 +744,10 @@ class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
744 def load_weights(self, weights: Iterable[Tuple[str,
745 torch.Tensor]]) -> Set[str]:
746 loader = AutoWeightsLoader(self, skip_prefixes=["proj_out."])
734 - mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
747 +
748 # add fake zeros bias for k_proj to state_dict
749 weights = _create_fake_bias_for_k_proj(weights)
737 - return loader.load_weights(weights, mapper=mapper)
750 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
751
752
753 def _create_fake_bias_for_k_proj(