A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc]Further reduce BNB static variable (#10597)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 27, 2024 at 14:54 UTC
15cc2a9f1acb70b68366da0a6d2a4549da3d32f4
14 files changed
+131
-219
vllm/model_executor/model_loader/loader.py
+130
-88
@@ -28,7 +28,8 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28
get_tensor_model_parallel_world_size)
29
from vllm.envs import VLLM_USE_MODELSCOPE
30
from vllm.logger import init_logger
31
-from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31
+from vllm.model_executor.layers.linear import (LinearBase,
32
+ MergedColumnParallelLinear,
33
QKVParallelLinear,
34
ReplicatedLinear,
35
RowParallelLinear)
@@ -78,12 +79,14 @@ def device_loading_context(module: torch.nn.Module,
79
original_device: torch.device = original_device_states[name]
80
if original_device.type == "cpu":
81
# `torch.empty_like` does not support `pin_memory` argument
81
- cpu_data = torch.empty_strided(size=p.data.size(),
82
- stride=p.data.stride(),
83
- dtype=p.data.dtype,
84
- layout=p.data.layout,
85
- device="cpu",
86
- pin_memory=pin_memory)
82
+ cpu_data = torch.empty_strided(
83
+ size=p.data.size(),
84
+ stride=p.data.stride(),
85
+ dtype=p.data.dtype,
86
+ layout=p.data.layout,
87
+ device="cpu",
88
+ pin_memory=pin_memory,
89
+ )
90
cpu_data.copy_(p.data)
91
p.data = cpu_data
92
else:
@@ -112,7 +115,8 @@ def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
115
logger.warning(msg)
116
logger.warning(
117
"Trying to guess the arguments for old-style model class %s",
115
- model_class)
118
+ model_class,
119
+ )
120
# try to be compatible with old-style model class
121
kwargs = {}
122
if "prefix" in all_params:
@@ -198,14 +202,17 @@ class DefaultModelLoader(BaseModelLoader):
202
return model_path
203
return None
204
201
- def _prepare_weights(self, model_name_or_path: str,
202
- revision: Optional[str],
203
- fall_back_to_pt: bool) -> Tuple[str, List[str], bool]:
205
+ def _prepare_weights(
206
+ self,
207
+ model_name_or_path: str,
208
+ revision: Optional[str],
209
+ fall_back_to_pt: bool,
210
+ ) -> Tuple[str, List[str], bool]:
211
"""Prepare weights for the model.
212
213
If the model is not local, it will be downloaded."""
207
- model_name_or_path = self._maybe_download_from_modelscope(
208
- model_name_or_path, revision) or model_name_or_path
214
+ model_name_or_path = (self._maybe_download_from_modelscope(
215
+ model_name_or_path, revision) or model_name_or_path)
216
217
is_local = os.path.isdir(model_name_or_path)
218
load_format = self.load_config.load_format
@@ -258,8 +265,11 @@ class DefaultModelLoader(BaseModelLoader):
265
# any files not found in the index.
266
if not is_local:
267
download_safetensors_index_file_from_hf(
261
- model_name_or_path, index_file,
262
- self.load_config.download_dir, revision)
268
+ model_name_or_path,
269
+ index_file,
270
+ self.load_config.download_dir,
271
+ revision,
272
+ )
273
hf_weights_files = filter_duplicate_safetensors_files(
274
hf_weights_files, hf_folder, index_file)
275
else:
@@ -282,8 +292,11 @@ class DefaultModelLoader(BaseModelLoader):
292
# Currently np_cache only support *.bin checkpoints
293
assert use_safetensors is False
294
weights_iterator = np_cache_weights_iterator(
285
- source.model_or_path, self.load_config.download_dir, hf_folder,
286
- hf_weights_files)
295
+ source.model_or_path,
296
+ self.load_config.download_dir,
297
+ hf_folder,
298
+ hf_weights_files,
299
+ )
300
elif use_safetensors:
301
weights_iterator = safetensors_weights_iterator(hf_weights_files)
302
else:
@@ -310,17 +323,19 @@ class DefaultModelLoader(BaseModelLoader):
323
model_config: ModelConfig,
324
model: nn.Module,
325
) -> Generator[Tuple[str, torch.Tensor], None, None]:
313
-
326
primary_weights = DefaultModelLoader.Source(
327
model_config.model,
328
model_config.revision,
329
prefix="",
330
fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load",
319
- True))
331
+ True),
332
+ )
333
yield from self._get_weights_iterator(primary_weights)
334
322
- secondary_weights = cast(Iterable[DefaultModelLoader.Source],
323
- getattr(model, "secondary_weights", ()))
335
+ secondary_weights = cast(
336
+ Iterable[DefaultModelLoader.Source],
337
+ getattr(model, "secondary_weights", ()),
338
+ )
339
for source in secondary_weights:
340
yield from self._get_weights_iterator(source)
341
@@ -416,7 +431,7 @@ class TensorizerLoader(BaseModelLoader):
431
self.tensorizer_config.verify_with_parallel_config(parallel_config)
432
433
def _get_weights_iterator(
419
- self) -> Generator[Tuple[str, torch.Tensor], None, None]:
434
+ self, ) -> Generator[Tuple[str, torch.Tensor], None, None]:
435
tensorizer_args = self.tensorizer_config._construct_tensorizer_args()
436
return tensorizer_weights_iterator(tensorizer_args)
437
@@ -479,9 +494,10 @@ class TensorizerLoader(BaseModelLoader):
494
495
if parallel_config.tensor_parallel_size > 1:
496
from vllm.distributed import get_tensor_model_parallel_rank
482
- self.tensorizer_config.tensorizer_uri = \
483
- self.tensorizer_config.tensorizer_uri \
484
- % get_tensor_model_parallel_rank()
497
+
498
+ self.tensorizer_config.tensorizer_uri = (
499
+ self.tensorizer_config.tensorizer_uri %
500
+ get_tensor_model_parallel_rank())
501
502
if is_vllm_tensorized(self.tensorizer_config):
503
return self._load_model_serialized(vllm_config=vllm_config)
@@ -520,13 +536,13 @@ class ShardedStateLoader(BaseModelLoader):
536
537
@staticmethod
538
def _filter_subtensors(
523
- tensors: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
539
+ tensors: Dict[str, torch.Tensor], ) -> Dict[str, torch.Tensor]:
540
"""
541
Filter out all tensors that share the same memory or a subset of the
542
memory of another tensor.
543
"""
528
- same_storage_groups: Dict[Any, List[Tuple[
529
- str, torch.Tensor]]] = collections.defaultdict(list)
544
+ same_storage_groups: Dict[Any, List[Tuple[str, torch.Tensor]]] = (
545
+ collections.defaultdict(list))
546
for key, tensor in tensors.items():
547
if tensor.numel():
548
ptr = tensor.untyped_storage().data_ptr()
@@ -615,8 +631,11 @@ class ShardedStateLoader(BaseModelLoader):
631
if tensor.shape != param_shape:
632
logger.warning(
633
"loading tensor of shape %s into "
618
- "parameter '%s' of shape %s", tensor.shape,
619
- key, param_shape)
634
+ "parameter '%s' of shape %s",
635
+ tensor.shape,
636
+ key,
637
+ param_shape,
638
+ )
639
param_data.copy_(tensor)
640
state_dict.pop(key)
641
if state_dict:
@@ -634,6 +653,7 @@ class ShardedStateLoader(BaseModelLoader):
653
from safetensors.torch import save_file
654
655
from vllm.distributed import get_tensor_model_parallel_rank
656
+
657
if pattern is None:
658
pattern = ShardedStateLoader.DEFAULT_PATTERN
659
rank = get_tensor_model_parallel_rank()
@@ -667,24 +687,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
687
688
possible_config_file_names = ["adapter_config.json"]
689
670
- default_target_modules = [
671
- ".gate_proj.",
672
- ".down_proj.",
673
- ".up_proj.",
674
- ".q_proj.",
675
- ".k_proj.",
676
- ".v_proj.",
677
- ".o_proj.",
678
- '.fc1.',
679
- '.fc2.',
680
- '.dense.',
681
- '.query_key_value.',
682
- '.qkv_proj.',
683
- '.dense_h_to_4h.',
684
- '.dense_4h_to_h.',
685
- '.out_proj.',
686
- ]
687
-
690
def __init__(self, load_config: LoadConfig):
691
super().__init__(load_config)
692
@@ -709,6 +711,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
711
with open(config_file_path) as f:
712
config = json.load(f)
713
self.target_modules = config["target_modules"]
714
+ # TODO: target_modules could be either a list or a regex string.
715
+ # We need to handle both cases.
716
+ assert isinstance(self.target_modules,
717
+ list), "Unsupported target_modules: "
718
+ f"{self.target_modules}"
719
720
def _get_config_file(self, qlora_adapter: str) -> str:
721
is_local = os.path.isdir(qlora_adapter)
@@ -734,12 +741,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
741
return config_file_path
742
743
def _get_weight_files(
737
- self,
738
- model_name_or_path: str,
739
- allowed_patterns: List[str],
740
- revision: Optional[str] = None) -> Tuple[List[str], str]:
741
- """Retrieve weight files. Download the files if necessary.
742
-
744
+ self,
745
+ model_name_or_path: str,
746
+ allowed_patterns: List[str],
747
+ revision: Optional[str] = None,
748
+ ) -> Tuple[List[str], str]:
749
+ """Retrieve weight files. Download the files if necessary.
750
+
751
Return the weight files and the file pattern."""
752
is_local = os.path.isdir(model_name_or_path)
753
@@ -806,6 +814,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
814
# only load the bitsandbytes module when needed
815
try:
816
import bitsandbytes
817
+
818
if bitsandbytes.__version__ < "0.44.0":
819
raise ImportError("bitsandbytes version is wrong. Please "
820
"install bitsandbytes>=0.44.0.")
@@ -839,8 +848,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
848
849
def _is_4bit_weight_name(self, weight_name: str):
850
quantized_suffix = {
842
- "absmax", "quant_map", "nested_absmax", "nested_quant_map",
843
- "bitsandbytes"
851
+ "absmax",
852
+ "quant_map",
853
+ "nested_absmax",
854
+ "nested_quant_map",
855
+ "bitsandbytes",
856
}
857
suffix = weight_name.split(".")[-1]
858
return any(q_suffix in suffix for q_suffix in quantized_suffix)
@@ -857,7 +869,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
869
870
for weight_name, weight_tensor in self._hf_weight_iter(
871
hf_weights_files, use_safetensors):
860
-
872
if self._is_8bit_weight_name(weight_name):
873
continue
874
@@ -899,14 +910,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
910
# pre quantized weights would have a quant_state
911
for weight_name, weight_tensor in self._hf_weight_iter(
912
hf_weights_files, use_safetensors):
902
-
913
if self._is_4bit_weight_name(weight_name):
914
continue
915
906
- if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
907
- in temp_state_dict) or \
908
- (f"{weight_name}.quant_state.bitsandbytes__fp4" \
909
- in temp_state_dict):
916
+ if (f"{weight_name}.quant_state.bitsandbytes__nf4"
917
+ in temp_state_dict) or (
918
+ f"{weight_name}.quant_state.bitsandbytes__fp4"
919
+ in temp_state_dict):
920
quant_state = _parse_quant_state(weight_name, temp_state_dict)
921
quant_state_dict[weight_name] = quant_state
922
yield weight_name, weight_tensor
@@ -916,12 +926,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
926
def _unquantized_generator(self, hf_weights_files, use_safetensors,
927
quant_state_dict) -> Generator:
928
from bitsandbytes.functional import quantize_4bit
929
+
930
tp_size = get_tensor_model_parallel_world_size()
931
tp_rank = get_tensor_model_parallel_rank()
932
933
for weight_name, weight_tensor in self._hf_weight_iter(
934
hf_weights_files, use_safetensors):
924
-
935
if any(target_module in weight_name for target_module in
936
self.target_modules) and weight_name.endswith(".weight"):
937
# Without sharding
@@ -954,12 +964,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
964
# get the start/end index of each shard weight tensor
965
total_start_index = list(
966
itertools.accumulate([0] + total_shard_sizes))[:-1]
957
- shard_weights_index = [
958
- (idx + size // tp_size * tp_rank,
959
- idx + size // tp_size * (tp_rank + 1))
960
- for idx, size in zip(total_start_index,
961
- total_shard_sizes)
962
- ]
967
+ shard_weights_index = [(
968
+ idx + size // tp_size * tp_rank,
969
+ idx + size // tp_size * (tp_rank + 1),
970
+ ) for idx, size in zip(total_start_index,
971
+ total_shard_sizes)]
972
# slice and reorder the weight tensor
973
weight_tensor = [
974
weight_tensor[start_index:end_index, ...]
@@ -989,7 +998,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
998
processed_weight, quant_state = quantize_4bit(
999
loaded_weight,
1000
compress_statistics=True,
992
- quant_type="nf4")
1001
+ quant_type="nf4",
1002
+ )
1003
1004
quant_state_dict[weight_name] = quant_state
1005
else:
@@ -997,28 +1007,58 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1007
1008
yield weight_name, processed_weight
1009
1010
+ def _get_bnb_target_modules(self, model: nn.Module) -> None:
1011
+
1012
+ # TODO: Maybe we can replace bitsandbytes_stacked_params_mapping with
1013
+ # packed_modules_mapping.
1014
+ inverse_stacked_mapping: Dict[str, List[str]] = {}
1015
+ for orig, (
1016
+ packed,
1017
+ idx,
1018
+ ) in model.bitsandbytes_stacked_params_mapping.items():
1019
+ if packed not in inverse_stacked_mapping:
1020
+ inverse_stacked_mapping[packed] = []
1021
+ inverse_stacked_mapping[packed].insert(idx, orig)
1022
+
1023
+ linear_module_lst = []
1024
+ for name, module in model.named_modules():
1025
+ if isinstance(module, (LinearBase, )):
1026
+ last_name = name.split(".")[-1]
1027
+ if sub_modules := inverse_stacked_mapping.get(last_name, []):
1028
+ # Map vllm's names to transformers' names.
1029
+ for sub_name in sub_modules:
1030
+ linear_module_lst.append(
1031
+ name.replace(last_name, sub_name))
1032
+ else:
1033
+ linear_module_lst.append(name)
1034
+ if self.target_modules:
1035
+ # Update self.target_modules
1036
+ self.target_modules = [
1037
+ qual_name for qual_name in linear_module_lst
1038
+ if any(t in qual_name for t in self.target_modules)
1039
+ ]
1040
+ else:
1041
+ self.target_modules = linear_module_lst
1042
+ assert (self.target_modules
1043
+ ), "vllm currently does not support BNB quantization for"
1044
+ f" {type(model).__name__}"
1045
+
1046
def _load_weights(self, model_config: ModelConfig,
1047
model: nn.Module) -> None:
1002
- if not hasattr(model, 'load_weights'):
1048
+ if not hasattr(model, "load_weights"):
1049
raise AttributeError(
1050
"The required method 'load_weights' is not defined in class"
1051
f" {type(model).__name__}.")
1052
1007
- if not hasattr(model, 'bitsandbytes_stacked_params_mapping'):
1053
+ if not hasattr(model, "bitsandbytes_stacked_params_mapping"):
1054
raise AttributeError(
1055
f"Model {type(model).__name__} does not support BitsAndBytes "
1056
"quantization yet.")
1057
1012
- if len(self.target_modules) == 0:
1013
- if hasattr(model, 'default_bitsandbytes_target_modules'):
1014
- self.target_modules = model.default_bitsandbytes_target_modules
1015
- else:
1016
- self.target_modules = self.default_target_modules
1017
-
1058
# Modules whose weights might have fused on disk
1059
# we need their output_sizes to make shard in flight correctly with TP
1060
self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
1021
-
1061
+ self._get_bnb_target_modules(model)
1062
for name, module in model.named_modules():
1063
# Some modules like `ReplicatedLinear` should not have their weights
1064
# sharded. The reason for implementing it this way is to avoid new
@@ -1046,7 +1086,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1086
1087
pre_quant = False
1088
if quant_config is not None:
1049
- quant_method = quant_config.get('quant_method')
1089
+ quant_method = quant_config.get("quant_method")
1090
if quant_method == "bitsandbytes":
1091
pre_quant = True
1092
else:
@@ -1063,11 +1103,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1103
1104
load_8bit = False
1105
if pre_quant:
1066
- load_8bit = quant_config.get('load_in_8bit', False)
1106
+ load_8bit = quant_config.get("load_in_8bit", False)
1107
1068
- qweight_iterator, quant_state_dict = \
1069
- self._get_quantized_weights_iterator(
1070
- model_config.model, model_config.revision, pre_quant, load_8bit)
1108
+ qweight_iterator, quant_state_dict = (
1109
+ self._get_quantized_weights_iterator(model_config.model,
1110
+ model_config.revision,
1111
+ pre_quant, load_8bit))
1112
1113
model.load_weights(qweight_iterator)
1114
@@ -1078,6 +1119,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1119
# TODO: Change this lazy import to normal import
1120
# after the checks are updated to run on a new version
1121
from vllm.model_executor.models.utils import is_pp_missing_parameter
1122
+
1123
for quant_param_name in quant_state_dict:
1124
if is_pp_missing_parameter(quant_param_name, model):
1125
continue
@@ -1086,9 +1128,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1128
1129
shard_index = 0
1130
for shard_name, (
1089
- weight_name, index
1131
+ weight_name,
1132
+ index,
1133
) in model.bitsandbytes_stacked_params_mapping.items():
1091
-
1134
shard_pos = quant_param_name.find(shard_name)
1135
# Some models, such as MiniCPM V2.5/2.6, contain both
1136
# module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
@@ -1123,8 +1165,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1165
1166
num_elements = [0] * len(quant_states)
1167
for seq, quant_state in quant_states.items():
1126
- num_elements[seq] = math.prod(
1127
- quant_state.shape) // pack_ratio
1168
+ num_elements[seq] = (math.prod(quant_state.shape) //
1169
+ pack_ratio)
1170
1171
offsets = np.concatenate(([0], np.cumsum(num_elements)))
1172
set_weight_attrs(param, {"bnb_shard_offsets": offsets})
vllm/model_executor/models/baichuan.py
-8
@@ -351,14 +351,6 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
351
embedding_padding_modules = []
352
353
# BitandBytes specific attributes
354
- default_bitsandbytes_target_modules = [
355
- ".W_pack.",
356
- ".o_proj.",
357
- ".down_proj.",
358
- ".up_proj.",
359
- ".gate_proj.",
360
- ".up_proj.",
361
- ]
354
bitsandbytes_stacked_params_mapping = {
355
# shard_name, weight_name, index
356
"gate_proj": ("gate_up_proj", 0),
vllm/model_executor/models/falcon.py
-6
@@ -412,12 +412,6 @@ class FalconForCausalLM(nn.Module, SupportsPP):
412
413
# BitandBytes specific attributes
414
bitsandbytes_stacked_params_mapping = {}
415
- default_bitsandbytes_target_modules = [
416
- ".query_key_value.",
417
- ".dense.",
418
- ".dense_h_to_4h.",
419
- ".dense_4h_to_h.",
420
- ]
415
416
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
417
super().__init__()
vllm/model_executor/models/gemma.py
-9
@@ -350,15 +350,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350
"down_proj",
351
]
352
# BitandBytes specific attributes
353
- default_bitsandbytes_target_modules = [
354
- ".gate_proj.",
355
- ".down_proj.",
356
- ".up_proj.",
357
- ".q_proj.",
358
- ".k_proj.",
359
- ".v_proj.",
360
- ".o_proj.",
361
- ]
353
bitsandbytes_stacked_params_mapping = {
354
# shard_name, weight_name, index
355
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/gemma2.py
-9
@@ -386,15 +386,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
386
embedding_padding_modules = []
387
388
# BitandBytes specific attributes
389
- default_bitsandbytes_target_modules = [
390
- ".gate_proj.",
391
- ".down_proj.",
392
- ".up_proj.",
393
- ".q_proj.",
394
- ".k_proj.",
395
- ".v_proj.",
396
- ".o_proj.",
397
- ]
389
bitsandbytes_stacked_params_mapping = {
390
# shard_name, weight_name, index
391
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/idefics3.py
-15
@@ -656,21 +656,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
656
]
657
658
# BitandBytes specific attributes
659
- default_bitsandbytes_target_modules = [
660
- ".gate_proj.",
661
- ".down_proj.",
662
- ".up_proj.",
663
- ".q_proj.",
664
- ".k_proj.",
665
- ".v_proj.",
666
- ".o_proj.",
667
- # vision_model
668
- ".fc1.",
669
- ".fc2.",
670
- ".out_proj.",
671
- # connector
672
- ".proj.",
673
- ]
659
bitsandbytes_stacked_params_mapping = {
660
# shard_name, weight_name, index
661
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/llama.py
-9
@@ -463,15 +463,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
463
embedding_padding_modules = ["lm_head"]
464
465
# BitandBytes specific attributes
466
- default_bitsandbytes_target_modules = [
467
- ".gate_proj.",
468
- ".down_proj.",
469
- ".up_proj.",
470
- ".q_proj.",
471
- ".k_proj.",
472
- ".v_proj.",
473
- ".o_proj.",
474
- ]
466
bitsandbytes_stacked_params_mapping = {
467
# shard_name, weight_name, index
468
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/minicpmv.py
-34
@@ -822,25 +822,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
822
]
823
824
# BitandBytes specific attributes
825
- default_bitsandbytes_target_modules = [
826
- ".gate_proj.",
827
- ".down_proj.",
828
- ".up_proj.",
829
- ".q_proj.",
830
- ".k_proj.",
831
- ".v_proj.",
832
- ".o_proj.",
833
- # vision encoder
834
- ".fc1.",
835
- ".fc2.",
836
- # Currently, vllm does not support BNB quantization for the `out_proj`
837
- # of the resampler, so it's necessary to distinguish between the
838
- # vision encoder and the resampler's out_proj. The same applies to
839
- # MiniCPMV2_6.
840
- ".self_attn.out_proj.", # vision encoder out_proj
841
- # resampler
842
- ".kv_proj.",
843
- ]
825
bitsandbytes_stacked_params_mapping = {
826
# shard_name, weight_name, index
827
"q_proj": ("qkv_proj", 0),
@@ -964,21 +945,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
945
]
946
947
# BitandBytes specific attributes
967
- default_bitsandbytes_target_modules = [
968
- ".gate_proj.",
969
- ".down_proj.",
970
- ".up_proj.",
971
- ".q_proj.",
972
- ".k_proj.",
973
- ".v_proj.",
974
- ".o_proj.",
975
- # vision encoder
976
- ".fc1.",
977
- ".fc2.",
978
- ".self_attn.out_proj.",
979
- # resampler
980
- ".kv_proj.",
981
- ]
948
bitsandbytes_stacked_params_mapping = {
949
# shard_name, weight_name, index
950
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/mllama.py
-14
@@ -1104,20 +1104,6 @@ class MllamaForCausalLM(nn.Module):
1104
@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1105
class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1106
# BitandBytes specific attributes
1107
- default_bitsandbytes_target_modules = [
1108
- ".gate_proj.",
1109
- ".down_proj.",
1110
- ".up_proj.",
1111
- ".q_proj.",
1112
- ".k_proj.",
1113
- ".v_proj.",
1114
- ".o_proj.",
1115
- ".fc1.",
1116
- ".fc2.",
1117
- # The `multi_modal_projector` is at the top level of the model,
1118
- # so we can't add a dot in front of it.
1119
- "multi_modal_projector."
1120
- ]
1107
bitsandbytes_stacked_params_mapping = {
1108
# shard_name, weight_name, index
1109
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/opt.py
-3
@@ -337,9 +337,6 @@ class OPTForCausalLM(nn.Module, SupportsPP):
337
"k_proj": ("qkv_proj", 1),
338
"v_proj": ("qkv_proj", 2),
339
}
340
- default_bitsandbytes_target_modules = [
341
- ".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
342
- ]
340
341
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
342
super().__init__()
vllm/model_executor/models/phi.py
-3
@@ -286,9 +286,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
286
"k_proj": ("qkv_proj", 1),
287
"v_proj": ("qkv_proj", 2),
288
}
289
- default_bitsandbytes_target_modules = [
290
- ".q_proj.", ".k_proj.", ".v_proj.", ".fc1.", ".fc2.", ".dense."
291
- ]
289
290
embedding_modules = {}
291
embedding_padding_modules = []
vllm/model_executor/models/phi3.py
-6
@@ -16,11 +16,5 @@ class Phi3ForCausalLM(LlamaForCausalLM):
16
}
17
18
# BitandBytes specific attributes
19
- default_bitsandbytes_target_modules = [
20
- ".gate_up_proj.",
21
- ".down_proj.",
22
- ".qkv_proj.",
23
- ".o_proj.",
24
- ]
19
# Initialize an empty dict when there is no stacked parameter mapping.
20
bitsandbytes_stacked_params_mapping = {}
vllm/model_executor/models/qwen.py
+1
-6
@@ -1028,12 +1028,7 @@ class QWenLLM(QWenBaseModel):
1028
embedding_modules = {}
1029
embedding_padding_modules = []
1030
1031
- default_bitsandbytes_target_modules = [
1032
- ".c_attn.",
1033
- ".c_proj.",
1034
- ".w1.",
1035
- ".w2.",
1036
- ]
1031
+ # BitandBytes specific attributes
1032
bitsandbytes_stacked_params_mapping = {
1033
# shard_name, weight_name, index
1034
"w2": ("gate_up_proj", 0),
vllm/model_executor/models/qwen2.py
-9
@@ -419,15 +419,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419
embedding_padding_modules = []
420
421
# BitandBytes specific attributes
422
- default_bitsandbytes_target_modules = [
423
- ".gate_proj.",
424
- ".down_proj.",
425
- ".up_proj.",
426
- ".q_proj.",
427
- ".k_proj.",
428
- ".v_proj.",
429
- ".o_proj.",
430
- ]
422
bitsandbytes_stacked_params_mapping = {
423
# shard_name, weight_name, index
424
"q_proj": ("qkv_proj", 0),