@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15cc2a9f1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc]Further reduce BNB static variable (#10597)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 27, 2024 at 14:54 UTC 15cc2a9f1acb70b68366da0a6d2a4549da3d32f4
14 files changed +131 -219
vllm/model_executor/model_loader/loader.py
+130 -88
@@ -28,7 +28,8 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.envs import VLLM_USE_MODELSCOPE
30 from vllm.logger import init_logger
31 -from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31 +from vllm.model_executor.layers.linear import (LinearBase,
32 + MergedColumnParallelLinear,
33 QKVParallelLinear,
34 ReplicatedLinear,
35 RowParallelLinear)
@@ -78,12 +79,14 @@ def device_loading_context(module: torch.nn.Module,
79 original_device: torch.device = original_device_states[name]
80 if original_device.type == "cpu":
81 # `torch.empty_like` does not support `pin_memory` argument
81 - cpu_data = torch.empty_strided(size=p.data.size(),
82 - stride=p.data.stride(),
83 - dtype=p.data.dtype,
84 - layout=p.data.layout,
85 - device="cpu",
86 - pin_memory=pin_memory)
82 + cpu_data = torch.empty_strided(
83 + size=p.data.size(),
84 + stride=p.data.stride(),
85 + dtype=p.data.dtype,
86 + layout=p.data.layout,
87 + device="cpu",
88 + pin_memory=pin_memory,
89 + )
90 cpu_data.copy_(p.data)
91 p.data = cpu_data
92 else:
@@ -112,7 +115,8 @@ def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
115 logger.warning(msg)
116 logger.warning(
117 "Trying to guess the arguments for old-style model class %s",
115 - model_class)
118 + model_class,
119 + )
120 # try to be compatible with old-style model class
121 kwargs = {}
122 if "prefix" in all_params:
@@ -198,14 +202,17 @@ class DefaultModelLoader(BaseModelLoader):
202 return model_path
203 return None
204
201 - def _prepare_weights(self, model_name_or_path: str,
202 - revision: Optional[str],
203 - fall_back_to_pt: bool) -> Tuple[str, List[str], bool]:
205 + def _prepare_weights(
206 + self,
207 + model_name_or_path: str,
208 + revision: Optional[str],
209 + fall_back_to_pt: bool,
210 + ) -> Tuple[str, List[str], bool]:
211 """Prepare weights for the model.
212
213 If the model is not local, it will be downloaded."""
207 - model_name_or_path = self._maybe_download_from_modelscope(
208 - model_name_or_path, revision) or model_name_or_path
214 + model_name_or_path = (self._maybe_download_from_modelscope(
215 + model_name_or_path, revision) or model_name_or_path)
216
217 is_local = os.path.isdir(model_name_or_path)
218 load_format = self.load_config.load_format
@@ -258,8 +265,11 @@ class DefaultModelLoader(BaseModelLoader):
265 # any files not found in the index.
266 if not is_local:
267 download_safetensors_index_file_from_hf(
261 - model_name_or_path, index_file,
262 - self.load_config.download_dir, revision)
268 + model_name_or_path,
269 + index_file,
270 + self.load_config.download_dir,
271 + revision,
272 + )
273 hf_weights_files = filter_duplicate_safetensors_files(
274 hf_weights_files, hf_folder, index_file)
275 else:
@@ -282,8 +292,11 @@ class DefaultModelLoader(BaseModelLoader):
292 # Currently np_cache only support *.bin checkpoints
293 assert use_safetensors is False
294 weights_iterator = np_cache_weights_iterator(
285 - source.model_or_path, self.load_config.download_dir, hf_folder,
286 - hf_weights_files)
295 + source.model_or_path,
296 + self.load_config.download_dir,
297 + hf_folder,
298 + hf_weights_files,
299 + )
300 elif use_safetensors:
301 weights_iterator = safetensors_weights_iterator(hf_weights_files)
302 else:
@@ -310,17 +323,19 @@ class DefaultModelLoader(BaseModelLoader):
323 model_config: ModelConfig,
324 model: nn.Module,
325 ) -> Generator[Tuple[str, torch.Tensor], None, None]:
313 -
326 primary_weights = DefaultModelLoader.Source(
327 model_config.model,
328 model_config.revision,
329 prefix="",
330 fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load",
319 - True))
331 + True),
332 + )
333 yield from self._get_weights_iterator(primary_weights)
334
322 - secondary_weights = cast(Iterable[DefaultModelLoader.Source],
323 - getattr(model, "secondary_weights", ()))
335 + secondary_weights = cast(
336 + Iterable[DefaultModelLoader.Source],
337 + getattr(model, "secondary_weights", ()),
338 + )
339 for source in secondary_weights:
340 yield from self._get_weights_iterator(source)
341
@@ -416,7 +431,7 @@ class TensorizerLoader(BaseModelLoader):
431 self.tensorizer_config.verify_with_parallel_config(parallel_config)
432
433 def _get_weights_iterator(
419 - self) -> Generator[Tuple[str, torch.Tensor], None, None]:
434 + self, ) -> Generator[Tuple[str, torch.Tensor], None, None]:
435 tensorizer_args = self.tensorizer_config._construct_tensorizer_args()
436 return tensorizer_weights_iterator(tensorizer_args)
437
@@ -479,9 +494,10 @@ class TensorizerLoader(BaseModelLoader):
494
495 if parallel_config.tensor_parallel_size > 1:
496 from vllm.distributed import get_tensor_model_parallel_rank
482 - self.tensorizer_config.tensorizer_uri = \
483 - self.tensorizer_config.tensorizer_uri \
484 - % get_tensor_model_parallel_rank()
497 +
498 + self.tensorizer_config.tensorizer_uri = (
499 + self.tensorizer_config.tensorizer_uri %
500 + get_tensor_model_parallel_rank())
501
502 if is_vllm_tensorized(self.tensorizer_config):
503 return self._load_model_serialized(vllm_config=vllm_config)
@@ -520,13 +536,13 @@ class ShardedStateLoader(BaseModelLoader):
536
537 @staticmethod
538 def _filter_subtensors(
523 - tensors: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
539 + tensors: Dict[str, torch.Tensor], ) -> Dict[str, torch.Tensor]:
540 """
541 Filter out all tensors that share the same memory or a subset of the
542 memory of another tensor.
543 """
528 - same_storage_groups: Dict[Any, List[Tuple[
529 - str, torch.Tensor]]] = collections.defaultdict(list)
544 + same_storage_groups: Dict[Any, List[Tuple[str, torch.Tensor]]] = (
545 + collections.defaultdict(list))
546 for key, tensor in tensors.items():
547 if tensor.numel():
548 ptr = tensor.untyped_storage().data_ptr()
@@ -615,8 +631,11 @@ class ShardedStateLoader(BaseModelLoader):
631 if tensor.shape != param_shape:
632 logger.warning(
633 "loading tensor of shape %s into "
618 - "parameter '%s' of shape %s", tensor.shape,
619 - key, param_shape)
634 + "parameter '%s' of shape %s",
635 + tensor.shape,
636 + key,
637 + param_shape,
638 + )
639 param_data.copy_(tensor)
640 state_dict.pop(key)
641 if state_dict:
@@ -634,6 +653,7 @@ class ShardedStateLoader(BaseModelLoader):
653 from safetensors.torch import save_file
654
655 from vllm.distributed import get_tensor_model_parallel_rank
656 +
657 if pattern is None:
658 pattern = ShardedStateLoader.DEFAULT_PATTERN
659 rank = get_tensor_model_parallel_rank()
@@ -667,24 +687,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
687
688 possible_config_file_names = ["adapter_config.json"]
689
670 - default_target_modules = [
671 - ".gate_proj.",
672 - ".down_proj.",
673 - ".up_proj.",
674 - ".q_proj.",
675 - ".k_proj.",
676 - ".v_proj.",
677 - ".o_proj.",
678 - '.fc1.',
679 - '.fc2.',
680 - '.dense.',
681 - '.query_key_value.',
682 - '.qkv_proj.',
683 - '.dense_h_to_4h.',
684 - '.dense_4h_to_h.',
685 - '.out_proj.',
686 - ]
687 -
690 def __init__(self, load_config: LoadConfig):
691 super().__init__(load_config)
692
@@ -709,6 +711,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
711 with open(config_file_path) as f:
712 config = json.load(f)
713 self.target_modules = config["target_modules"]
714 + # TODO: target_modules could be either a list or a regex string.
715 + # We need to handle both cases.
716 + assert isinstance(self.target_modules,
717 + list), "Unsupported target_modules: "
718 + f"{self.target_modules}"
719
720 def _get_config_file(self, qlora_adapter: str) -> str:
721 is_local = os.path.isdir(qlora_adapter)
@@ -734,12 +741,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
741 return config_file_path
742
743 def _get_weight_files(
737 - self,
738 - model_name_or_path: str,
739 - allowed_patterns: List[str],
740 - revision: Optional[str] = None) -> Tuple[List[str], str]:
741 - """Retrieve weight files. Download the files if necessary.
742 -
744 + self,
745 + model_name_or_path: str,
746 + allowed_patterns: List[str],
747 + revision: Optional[str] = None,
748 + ) -> Tuple[List[str], str]:
749 + """Retrieve weight files. Download the files if necessary.
750 +
751 Return the weight files and the file pattern."""
752 is_local = os.path.isdir(model_name_or_path)
753
@@ -806,6 +814,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
814 # only load the bitsandbytes module when needed
815 try:
816 import bitsandbytes
817 +
818 if bitsandbytes.__version__ < "0.44.0":
819 raise ImportError("bitsandbytes version is wrong. Please "
820 "install bitsandbytes>=0.44.0.")
@@ -839,8 +848,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
848
849 def _is_4bit_weight_name(self, weight_name: str):
850 quantized_suffix = {
842 - "absmax", "quant_map", "nested_absmax", "nested_quant_map",
843 - "bitsandbytes"
851 + "absmax",
852 + "quant_map",
853 + "nested_absmax",
854 + "nested_quant_map",
855 + "bitsandbytes",
856 }
857 suffix = weight_name.split(".")[-1]
858 return any(q_suffix in suffix for q_suffix in quantized_suffix)
@@ -857,7 +869,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
869
870 for weight_name, weight_tensor in self._hf_weight_iter(
871 hf_weights_files, use_safetensors):
860 -
872 if self._is_8bit_weight_name(weight_name):
873 continue
874
@@ -899,14 +910,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
910 # pre quantized weights would have a quant_state
911 for weight_name, weight_tensor in self._hf_weight_iter(
912 hf_weights_files, use_safetensors):
902 -
913 if self._is_4bit_weight_name(weight_name):
914 continue
915
906 - if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
907 - in temp_state_dict) or \
908 - (f"{weight_name}.quant_state.bitsandbytes__fp4" \
909 - in temp_state_dict):
916 + if (f"{weight_name}.quant_state.bitsandbytes__nf4"
917 + in temp_state_dict) or (
918 + f"{weight_name}.quant_state.bitsandbytes__fp4"
919 + in temp_state_dict):
920 quant_state = _parse_quant_state(weight_name, temp_state_dict)
921 quant_state_dict[weight_name] = quant_state
922 yield weight_name, weight_tensor
@@ -916,12 +926,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
926 def _unquantized_generator(self, hf_weights_files, use_safetensors,
927 quant_state_dict) -> Generator:
928 from bitsandbytes.functional import quantize_4bit
929 +
930 tp_size = get_tensor_model_parallel_world_size()
931 tp_rank = get_tensor_model_parallel_rank()
932
933 for weight_name, weight_tensor in self._hf_weight_iter(
934 hf_weights_files, use_safetensors):
924 -
935 if any(target_module in weight_name for target_module in
936 self.target_modules) and weight_name.endswith(".weight"):
937 # Without sharding
@@ -954,12 +964,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
964 # get the start/end index of each shard weight tensor
965 total_start_index = list(
966 itertools.accumulate([0] + total_shard_sizes))[:-1]
957 - shard_weights_index = [
958 - (idx + size // tp_size * tp_rank,
959 - idx + size // tp_size * (tp_rank + 1))
960 - for idx, size in zip(total_start_index,
961 - total_shard_sizes)
962 - ]
967 + shard_weights_index = [(
968 + idx + size // tp_size * tp_rank,
969 + idx + size // tp_size * (tp_rank + 1),
970 + ) for idx, size in zip(total_start_index,
971 + total_shard_sizes)]
972 # slice and reorder the weight tensor
973 weight_tensor = [
974 weight_tensor[start_index:end_index, ...]
@@ -989,7 +998,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
998 processed_weight, quant_state = quantize_4bit(
999 loaded_weight,
1000 compress_statistics=True,
992 - quant_type="nf4")
1001 + quant_type="nf4",
1002 + )
1003
1004 quant_state_dict[weight_name] = quant_state
1005 else:
@@ -997,28 +1007,58 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1007
1008 yield weight_name, processed_weight
1009
1010 + def _get_bnb_target_modules(self, model: nn.Module) -> None:
1011 +
1012 + # TODO: Maybe we can replace bitsandbytes_stacked_params_mapping with
1013 + # packed_modules_mapping.
1014 + inverse_stacked_mapping: Dict[str, List[str]] = {}
1015 + for orig, (
1016 + packed,
1017 + idx,
1018 + ) in model.bitsandbytes_stacked_params_mapping.items():
1019 + if packed not in inverse_stacked_mapping:
1020 + inverse_stacked_mapping[packed] = []
1021 + inverse_stacked_mapping[packed].insert(idx, orig)
1022 +
1023 + linear_module_lst = []
1024 + for name, module in model.named_modules():
1025 + if isinstance(module, (LinearBase, )):
1026 + last_name = name.split(".")[-1]
1027 + if sub_modules := inverse_stacked_mapping.get(last_name, []):
1028 + # Map vllm's names to transformers' names.
1029 + for sub_name in sub_modules:
1030 + linear_module_lst.append(
1031 + name.replace(last_name, sub_name))
1032 + else:
1033 + linear_module_lst.append(name)
1034 + if self.target_modules:
1035 + # Update self.target_modules
1036 + self.target_modules = [
1037 + qual_name for qual_name in linear_module_lst
1038 + if any(t in qual_name for t in self.target_modules)
1039 + ]
1040 + else:
1041 + self.target_modules = linear_module_lst
1042 + assert (self.target_modules
1043 + ), "vllm currently does not support BNB quantization for"
1044 + f" {type(model).__name__}"
1045 +
1046 def _load_weights(self, model_config: ModelConfig,
1047 model: nn.Module) -> None:
1002 - if not hasattr(model, 'load_weights'):
1048 + if not hasattr(model, "load_weights"):
1049 raise AttributeError(
1050 "The required method 'load_weights' is not defined in class"
1051 f" {type(model).__name__}.")
1052
1007 - if not hasattr(model, 'bitsandbytes_stacked_params_mapping'):
1053 + if not hasattr(model, "bitsandbytes_stacked_params_mapping"):
1054 raise AttributeError(
1055 f"Model {type(model).__name__} does not support BitsAndBytes "
1056 "quantization yet.")
1057
1012 - if len(self.target_modules) == 0:
1013 - if hasattr(model, 'default_bitsandbytes_target_modules'):
1014 - self.target_modules = model.default_bitsandbytes_target_modules
1015 - else:
1016 - self.target_modules = self.default_target_modules
1017 -
1058 # Modules whose weights might have fused on disk
1059 # we need their output_sizes to make shard in flight correctly with TP
1060 self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
1021 -
1061 + self._get_bnb_target_modules(model)
1062 for name, module in model.named_modules():
1063 # Some modules like `ReplicatedLinear` should not have their weights
1064 # sharded. The reason for implementing it this way is to avoid new
@@ -1046,7 +1086,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1086
1087 pre_quant = False
1088 if quant_config is not None:
1049 - quant_method = quant_config.get('quant_method')
1089 + quant_method = quant_config.get("quant_method")
1090 if quant_method == "bitsandbytes":
1091 pre_quant = True
1092 else:
@@ -1063,11 +1103,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1103
1104 load_8bit = False
1105 if pre_quant:
1066 - load_8bit = quant_config.get('load_in_8bit', False)
1106 + load_8bit = quant_config.get("load_in_8bit", False)
1107
1068 - qweight_iterator, quant_state_dict = \
1069 - self._get_quantized_weights_iterator(
1070 - model_config.model, model_config.revision, pre_quant, load_8bit)
1108 + qweight_iterator, quant_state_dict = (
1109 + self._get_quantized_weights_iterator(model_config.model,
1110 + model_config.revision,
1111 + pre_quant, load_8bit))
1112
1113 model.load_weights(qweight_iterator)
1114
@@ -1078,6 +1119,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1119 # TODO: Change this lazy import to normal import
1120 # after the checks are updated to run on a new version
1121 from vllm.model_executor.models.utils import is_pp_missing_parameter
1122 +
1123 for quant_param_name in quant_state_dict:
1124 if is_pp_missing_parameter(quant_param_name, model):
1125 continue
@@ -1086,9 +1128,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1128
1129 shard_index = 0
1130 for shard_name, (
1089 - weight_name, index
1131 + weight_name,
1132 + index,
1133 ) in model.bitsandbytes_stacked_params_mapping.items():
1091 -
1134 shard_pos = quant_param_name.find(shard_name)
1135 # Some models, such as MiniCPM V2.5/2.6, contain both
1136 # module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
@@ -1123,8 +1165,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1165
1166 num_elements = [0] * len(quant_states)
1167 for seq, quant_state in quant_states.items():
1126 - num_elements[seq] = math.prod(
1127 - quant_state.shape) // pack_ratio
1168 + num_elements[seq] = (math.prod(quant_state.shape) //
1169 + pack_ratio)
1170
1171 offsets = np.concatenate(([0], np.cumsum(num_elements)))
1172 set_weight_attrs(param, {"bnb_shard_offsets": offsets})
vllm/model_executor/models/baichuan.py
-8
@@ -351,14 +351,6 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
351 embedding_padding_modules = []
352
353 # BitandBytes specific attributes
354 - default_bitsandbytes_target_modules = [
355 - ".W_pack.",
356 - ".o_proj.",
357 - ".down_proj.",
358 - ".up_proj.",
359 - ".gate_proj.",
360 - ".up_proj.",
361 - ]
354 bitsandbytes_stacked_params_mapping = {
355 # shard_name, weight_name, index
356 "gate_proj": ("gate_up_proj", 0),
vllm/model_executor/models/falcon.py
-6
@@ -412,12 +412,6 @@ class FalconForCausalLM(nn.Module, SupportsPP):
412
413 # BitandBytes specific attributes
414 bitsandbytes_stacked_params_mapping = {}
415 - default_bitsandbytes_target_modules = [
416 - ".query_key_value.",
417 - ".dense.",
418 - ".dense_h_to_4h.",
419 - ".dense_4h_to_h.",
420 - ]
415
416 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
417 super().__init__()
vllm/model_executor/models/gemma.py
-9
@@ -350,15 +350,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350 "down_proj",
351 ]
352 # BitandBytes specific attributes
353 - default_bitsandbytes_target_modules = [
354 - ".gate_proj.",
355 - ".down_proj.",
356 - ".up_proj.",
357 - ".q_proj.",
358 - ".k_proj.",
359 - ".v_proj.",
360 - ".o_proj.",
361 - ]
353 bitsandbytes_stacked_params_mapping = {
354 # shard_name, weight_name, index
355 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/gemma2.py
-9
@@ -386,15 +386,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
386 embedding_padding_modules = []
387
388 # BitandBytes specific attributes
389 - default_bitsandbytes_target_modules = [
390 - ".gate_proj.",
391 - ".down_proj.",
392 - ".up_proj.",
393 - ".q_proj.",
394 - ".k_proj.",
395 - ".v_proj.",
396 - ".o_proj.",
397 - ]
389 bitsandbytes_stacked_params_mapping = {
390 # shard_name, weight_name, index
391 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/idefics3.py
-15
@@ -656,21 +656,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
656 ]
657
658 # BitandBytes specific attributes
659 - default_bitsandbytes_target_modules = [
660 - ".gate_proj.",
661 - ".down_proj.",
662 - ".up_proj.",
663 - ".q_proj.",
664 - ".k_proj.",
665 - ".v_proj.",
666 - ".o_proj.",
667 - # vision_model
668 - ".fc1.",
669 - ".fc2.",
670 - ".out_proj.",
671 - # connector
672 - ".proj.",
673 - ]
659 bitsandbytes_stacked_params_mapping = {
660 # shard_name, weight_name, index
661 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/llama.py
-9
@@ -463,15 +463,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
463 embedding_padding_modules = ["lm_head"]
464
465 # BitandBytes specific attributes
466 - default_bitsandbytes_target_modules = [
467 - ".gate_proj.",
468 - ".down_proj.",
469 - ".up_proj.",
470 - ".q_proj.",
471 - ".k_proj.",
472 - ".v_proj.",
473 - ".o_proj.",
474 - ]
466 bitsandbytes_stacked_params_mapping = {
467 # shard_name, weight_name, index
468 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/minicpmv.py
-34
@@ -822,25 +822,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
822 ]
823
824 # BitandBytes specific attributes
825 - default_bitsandbytes_target_modules = [
826 - ".gate_proj.",
827 - ".down_proj.",
828 - ".up_proj.",
829 - ".q_proj.",
830 - ".k_proj.",
831 - ".v_proj.",
832 - ".o_proj.",
833 - # vision encoder
834 - ".fc1.",
835 - ".fc2.",
836 - # Currently, vllm does not support BNB quantization for the `out_proj`
837 - # of the resampler, so it's necessary to distinguish between the
838 - # vision encoder and the resampler's out_proj. The same applies to
839 - # MiniCPMV2_6.
840 - ".self_attn.out_proj.", # vision encoder out_proj
841 - # resampler
842 - ".kv_proj.",
843 - ]
825 bitsandbytes_stacked_params_mapping = {
826 # shard_name, weight_name, index
827 "q_proj": ("qkv_proj", 0),
@@ -964,21 +945,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
945 ]
946
947 # BitandBytes specific attributes
967 - default_bitsandbytes_target_modules = [
968 - ".gate_proj.",
969 - ".down_proj.",
970 - ".up_proj.",
971 - ".q_proj.",
972 - ".k_proj.",
973 - ".v_proj.",
974 - ".o_proj.",
975 - # vision encoder
976 - ".fc1.",
977 - ".fc2.",
978 - ".self_attn.out_proj.",
979 - # resampler
980 - ".kv_proj.",
981 - ]
948 bitsandbytes_stacked_params_mapping = {
949 # shard_name, weight_name, index
950 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/mllama.py
-14
@@ -1104,20 +1104,6 @@ class MllamaForCausalLM(nn.Module):
1104 @INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1105 class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1106 # BitandBytes specific attributes
1107 - default_bitsandbytes_target_modules = [
1108 - ".gate_proj.",
1109 - ".down_proj.",
1110 - ".up_proj.",
1111 - ".q_proj.",
1112 - ".k_proj.",
1113 - ".v_proj.",
1114 - ".o_proj.",
1115 - ".fc1.",
1116 - ".fc2.",
1117 - # The `multi_modal_projector` is at the top level of the model,
1118 - # so we can't add a dot in front of it.
1119 - "multi_modal_projector."
1120 - ]
1107 bitsandbytes_stacked_params_mapping = {
1108 # shard_name, weight_name, index
1109 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/opt.py
-3
@@ -337,9 +337,6 @@ class OPTForCausalLM(nn.Module, SupportsPP):
337 "k_proj": ("qkv_proj", 1),
338 "v_proj": ("qkv_proj", 2),
339 }
340 - default_bitsandbytes_target_modules = [
341 - ".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
342 - ]
340
341 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
342 super().__init__()
vllm/model_executor/models/phi.py
-3
@@ -286,9 +286,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
286 "k_proj": ("qkv_proj", 1),
287 "v_proj": ("qkv_proj", 2),
288 }
289 - default_bitsandbytes_target_modules = [
290 - ".q_proj.", ".k_proj.", ".v_proj.", ".fc1.", ".fc2.", ".dense."
291 - ]
289
290 embedding_modules = {}
291 embedding_padding_modules = []
vllm/model_executor/models/phi3.py
-6
@@ -16,11 +16,5 @@ class Phi3ForCausalLM(LlamaForCausalLM):
16 }
17
18 # BitandBytes specific attributes
19 - default_bitsandbytes_target_modules = [
20 - ".gate_up_proj.",
21 - ".down_proj.",
22 - ".qkv_proj.",
23 - ".o_proj.",
24 - ]
19 # Initialize an empty dict when there is no stacked parameter mapping.
20 bitsandbytes_stacked_params_mapping = {}
vllm/model_executor/models/qwen.py
+1 -6
@@ -1028,12 +1028,7 @@ class QWenLLM(QWenBaseModel):
1028 embedding_modules = {}
1029 embedding_padding_modules = []
1030
1031 - default_bitsandbytes_target_modules = [
1032 - ".c_attn.",
1033 - ".c_proj.",
1034 - ".w1.",
1035 - ".w2.",
1036 - ]
1031 + # BitandBytes specific attributes
1032 bitsandbytes_stacked_params_mapping = {
1033 # shard_name, weight_name, index
1034 "w2": ("gate_up_proj", 0),
vllm/model_executor/models/qwen2.py
-9
@@ -419,15 +419,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419 embedding_padding_modules = []
420
421 # BitandBytes specific attributes
422 - default_bitsandbytes_target_modules = [
423 - ".gate_proj.",
424 - ".down_proj.",
425 - ".up_proj.",
426 - ".q_proj.",
427 - ".k_proj.",
428 - ".v_proj.",
429 - ".o_proj.",
430 - ]
422 bitsandbytes_stacked_params_mapping = {
423 # shard_name, weight_name, index
424 "q_proj": ("qkv_proj", 0),