A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc] Merge bitsandbytes_stacked_params_mapping and packed_modules_mapping (#11924)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 15, 2025 at 07:49 UTC
a3a3ee4e6febe8c270fdec0765c844186a728079
24 files changed
+49
-200
vllm/model_executor/model_loader/loader.py
+11
-16
@@ -39,7 +39,8 @@ from vllm.model_executor.layers.quantization.base_config import (
39
from vllm.model_executor.model_loader.tensorizer import (
40
TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
41
serialize_vllm_model, tensorizer_weights_iterator)
42
-from vllm.model_executor.model_loader.utils import (get_model_architecture,
42
+from vllm.model_executor.model_loader.utils import (ParamMapping,
43
+ get_model_architecture,
44
set_default_torch_dtype)
45
from vllm.model_executor.model_loader.weight_utils import (
46
download_safetensors_index_file_from_hf, download_weights_from_hf,
@@ -983,21 +984,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
984
985
def _get_bnb_target_modules(self, model: nn.Module) -> None:
986
986
- # TODO: Maybe we can replace bitsandbytes_stacked_params_mapping with
987
- # packed_modules_mapping.
988
- inverse_stacked_mapping: Dict[str, List[str]] = {}
989
- for orig, (
990
- packed,
991
- idx,
992
- ) in model.bitsandbytes_stacked_params_mapping.items():
993
- if packed not in inverse_stacked_mapping:
994
- inverse_stacked_mapping[packed] = []
995
- inverse_stacked_mapping[packed].insert(idx, orig)
996
-
987
for name, module in model.named_modules():
988
if isinstance(module, (LinearBase, )):
989
last_name = name.split(".")[-1]
1000
- if sub_modules := inverse_stacked_mapping.get(last_name, []):
990
+ if sub_modules := self.modules_mapping.packed_mapping.get(
991
+ last_name, []):
992
# Map vllm's names to transformers's names.
993
for sub_name in sub_modules:
994
self.target_modules.append(
@@ -1018,15 +1009,19 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1009
"The required method 'load_weights' is not defined in class"
1010
f" {type(model).__name__}.")
1011
1021
- if not hasattr(model, "bitsandbytes_stacked_params_mapping"):
1012
+ if not hasattr(model, "packed_modules_mapping"):
1013
raise AttributeError(
1014
f"Model {type(model).__name__} does not support BitsAndBytes "
1024
- "quantization yet.")
1015
+ "quantization yet. No 'packed_modules_mapping' found.")
1016
+
1017
+ self.modules_mapping = ParamMapping(
1018
+ copy.deepcopy(model.packed_modules_mapping))
1019
1020
# For some models like Molmo, we need to use hf_to_vllm_mapper
1021
# to ensure correct loading of weights.
1022
if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):
1023
self.weight_mapper = lambda name: hf_to_vllm_mapper._map_name(name)
1024
+
1025
# Modules whose weights might have fused on disk
1026
# we need their output_sizes to make shard in flight correctly with TP
1027
self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
@@ -1109,7 +1104,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1104
for shard_name, (
1105
weight_name,
1106
index,
1112
- ) in model.bitsandbytes_stacked_params_mapping.items():
1107
+ ) in self.modules_mapping.inverse_packed_mapping.items():
1108
shard_pos = quant_param_name.find(shard_name)
1109
# Some models, such as MiniCPM V2.5/2.6, contain both
1110
# module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
vllm/model_executor/model_loader/utils.py
+25
-1
@@ -1,6 +1,7 @@
1
"""Utilities for selecting and loading models."""
2
import contextlib
3
-from typing import Tuple, Type
3
+from dataclasses import dataclass, field
4
+from typing import Dict, List, Tuple, Type
5
6
import torch
7
from torch import nn
@@ -49,3 +50,26 @@ def get_model_architecture(
50
51
def get_architecture_class_name(model_config: ModelConfig) -> str:
52
return get_model_architecture(model_config)[1]
53
+
54
+
55
+@dataclass
56
+class ParamMapping:
57
+ """
58
+ A class to handle parameter mapping for model weight loading.
59
+ It creates a bidirectional mapping between packed parameters and their
60
+ constituent parts.
61
+ """
62
+ packed_mapping: Dict[str, List[str]]
63
+ inverse_packed_mapping: Dict[str, Tuple[str,
64
+ int]] = field(default_factory=dict)
65
+
66
+ def __post_init__(self):
67
+ for packed_name, sub_params in self.packed_mapping.items():
68
+ # Skip self-contained cases (e.g., {"W_pack": ["W_pack"]})
69
+ if len(sub_params) == 1 and sub_params[0] == packed_name:
70
+ continue
71
+ for index, param_name in enumerate(sub_params):
72
+ self.inverse_packed_mapping[param_name] = (
73
+ packed_name,
74
+ index,
75
+ )
vllm/model_executor/models/baichuan.py
-7
@@ -350,13 +350,6 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350
embedding_modules = {}
351
embedding_padding_modules = []
352
353
- # BitandBytes specific attributes
354
- bitsandbytes_stacked_params_mapping = {
355
- # shard_name, weight_name, index
356
- "gate_proj": ("gate_up_proj", 0),
357
- "up_proj": ("gate_up_proj", 1),
358
- }
359
-
353
def __init__(
354
self,
355
*,
vllm/model_executor/models/exaone.py
-8
@@ -430,14 +430,6 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
430
"lm_head": "output_embeddings",
431
}
432
embedding_padding_modules = ["lm_head"]
433
- bitsandbytes_stacked_params_mapping = {
434
- # shard_name, weight_name, index
435
- "q_proj": ("qkv_proj", 0),
436
- "k_proj": ("qkv_proj", 1),
437
- "v_proj": ("qkv_proj", 2),
438
- "c_fc_0": ("gate_up_proj", 0),
439
- "c_fc_1": ("gate_up_proj", 1),
440
- }
433
434
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
435
super().__init__()
vllm/model_executor/models/falcon.py
+3
-3
@@ -409,9 +409,9 @@ class FalconModel(nn.Module):
409
410
411
class FalconForCausalLM(nn.Module, SupportsPP):
412
-
413
- # BitandBytes specific attributes
414
- bitsandbytes_stacked_params_mapping = {}
412
+ packed_modules_mapping = {
413
+ "query_key_value": ["query_key_value"],
414
+ }
415
416
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
417
super().__init__()
vllm/model_executor/models/gemma.py
-9
@@ -349,15 +349,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
349
"gate_up_proj",
350
"down_proj",
351
]
352
- # BitandBytes specific attributes
353
- bitsandbytes_stacked_params_mapping = {
354
- # shard_name, weight_name, index
355
- "q_proj": ("qkv_proj", 0),
356
- "k_proj": ("qkv_proj", 1),
357
- "v_proj": ("qkv_proj", 2),
358
- "gate_proj": ("gate_up_proj", 0),
359
- "up_proj": ("gate_up_proj", 1),
360
- }
352
353
# Gemma does not apply LoRA to the embedding layer.
354
embedding_modules = {}
vllm/model_executor/models/gemma2.py
-10
@@ -399,16 +399,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
399
embedding_modules = {}
400
embedding_padding_modules = []
401
402
- # BitandBytes specific attributes
403
- bitsandbytes_stacked_params_mapping = {
404
- # shard_name, weight_name, index
405
- "q_proj": ("qkv_proj", 0),
406
- "k_proj": ("qkv_proj", 1),
407
- "v_proj": ("qkv_proj", 2),
408
- "gate_proj": ("gate_up_proj", 0),
409
- "up_proj": ("gate_up_proj", 1),
410
- }
411
-
402
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
403
config = vllm_config.model_config.hf_config
404
quant_config = vllm_config.quant_config
vllm/model_executor/models/granite.py
-8
@@ -362,14 +362,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
362
"lm_head": "output_embeddings",
363
}
364
embedding_padding_modules = ["lm_head"]
365
- bitsandbytes_stacked_params_mapping = {
366
- # shard_name, weight_name, index
367
- "q_proj": ("qkv_proj", 0),
368
- "k_proj": ("qkv_proj", 1),
369
- "v_proj": ("qkv_proj", 2),
370
- "gate_proj": ("gate_up_proj", 0),
371
- "up_proj": ("gate_up_proj", 1),
372
- }
365
366
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
367
super().__init__()
vllm/model_executor/models/idefics3.py
-10
@@ -662,16 +662,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
662
"down_proj",
663
]
664
665
- # BitandBytes specific attributes
666
- bitsandbytes_stacked_params_mapping = {
667
- # shard_name, weight_name, index
668
- "q_proj": ("qkv_proj", 0),
669
- "k_proj": ("qkv_proj", 1),
670
- "v_proj": ("qkv_proj", 2),
671
- "gate_proj": ("gate_up_proj", 0),
672
- "up_proj": ("gate_up_proj", 1),
673
- }
674
-
665
embedding_modules = {}
666
embedding_padding_modules = []
667
vllm/model_executor/models/llama.py
-10
@@ -478,16 +478,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
478
}
479
embedding_padding_modules = ["lm_head"]
480
481
- # BitandBytes specific attributes
482
- bitsandbytes_stacked_params_mapping = {
483
- # shard_name, weight_name, index
484
- "q_proj": ("qkv_proj", 0),
485
- "k_proj": ("qkv_proj", 1),
486
- "v_proj": ("qkv_proj", 2),
487
- "gate_proj": ("gate_up_proj", 0),
488
- "up_proj": ("gate_up_proj", 1),
489
- }
490
-
481
# Mistral/Llama models can also be loaded with --load-format mistral
482
# from consolidated.safetensors checkpoints
483
mistral_mapping = {
vllm/model_executor/models/llava.py
+4
-8
@@ -463,14 +463,10 @@ def init_vision_tower_for_llava(
463
info=_build_llava_or_pixtral_hf_info,
464
dummy_inputs=LlavaDummyInputsBuilder)
465
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
466
- # BitandBytes specific attributes
467
- bitsandbytes_stacked_params_mapping = {
468
- # shard_name, weight_name, index
469
- "q_proj": ("qkv_proj", 0),
470
- "k_proj": ("qkv_proj", 1),
471
- "v_proj": ("qkv_proj", 2),
472
- "gate_proj": ("gate_up_proj", 0),
473
- "up_proj": ("gate_up_proj", 1),
466
+
467
+ packed_modules_mapping = {
468
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
469
+ "gate_up_proj": ["gate_proj", "up_proj"]
470
}
471
472
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
vllm/model_executor/models/minicpm.py
-10
@@ -534,16 +534,6 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
534
}
535
embedding_padding_modules = ["lm_head"]
536
537
- # BitandBytes specific attributes
538
- bitsandbytes_stacked_params_mapping = {
539
- # shard_name, weight_name, index
540
- "q_proj": ("qkv_proj", 0),
541
- "k_proj": ("qkv_proj", 1),
542
- "v_proj": ("qkv_proj", 2),
543
- "gate_proj": ("gate_up_proj", 0),
544
- "up_proj": ("gate_up_proj", 1),
545
- }
546
-
537
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
538
super().__init__()
539
config = vllm_config.model_config.hf_config
vllm/model_executor/models/minicpm3.py
-6
@@ -241,11 +241,5 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
241
# `embedding_modules` and `embedding_padding_modules`
242
# are inherited from MiniCPMForCausalLM
243
244
- bitsandbytes_stacked_params_mapping = {
245
- # shard_name, weight_name, index
246
- "gate_proj": ("gate_up_proj", 0),
247
- "up_proj": ("gate_up_proj", 1),
248
- }
249
-
244
def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
245
return MiniCPM3Model(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/minicpmv.py
-20
@@ -761,16 +761,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
761
"kv_proj",
762
]
763
764
- # BitandBytes specific attributes
765
- bitsandbytes_stacked_params_mapping = {
766
- # shard_name, weight_name, index
767
- "q_proj": ("qkv_proj", 0),
768
- "k_proj": ("qkv_proj", 1),
769
- "v_proj": ("qkv_proj", 2),
770
- "gate_proj": ("gate_up_proj", 0),
771
- "up_proj": ("gate_up_proj", 1),
772
- }
773
-
764
embedding_modules = {}
765
embedding_padding_modules = []
766
@@ -881,16 +871,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
871
"kv_proj",
872
]
873
884
- # BitandBytes specific attributes
885
- bitsandbytes_stacked_params_mapping = {
886
- # shard_name, weight_name, index
887
- "q_proj": ("qkv_proj", 0),
888
- "k_proj": ("qkv_proj", 1),
889
- "v_proj": ("qkv_proj", 2),
890
- "gate_proj": ("gate_up_proj", 0),
891
- "up_proj": ("gate_up_proj", 1),
892
- }
893
-
874
embedding_modules = {}
875
embedding_padding_modules = []
876
vllm/model_executor/models/mllama.py
+3
-8
@@ -1107,14 +1107,9 @@ class MllamaForCausalLM(nn.Module):
1107
@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
1108
@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1109
class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1110
- # BitandBytes specific attributes
1111
- bitsandbytes_stacked_params_mapping = {
1112
- # shard_name, weight_name, index
1113
- "q_proj": ("qkv_proj", 0),
1114
- "k_proj": ("qkv_proj", 1),
1115
- "v_proj": ("qkv_proj", 2),
1116
- "gate_proj": ("gate_up_proj", 0),
1117
- "up_proj": ("gate_up_proj", 1),
1110
+ packed_modules_mapping = {
1111
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
1112
+ "gate_up_proj": ["gate_proj", "up_proj"]
1113
}
1114
1115
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/molmo.py
-6
@@ -1193,12 +1193,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1193
embedding_modules = {}
1194
embedding_padding_modules = []
1195
1196
- # BitandBytes specific attributes
1197
- bitsandbytes_stacked_params_mapping = {
1198
- "gate_proj": ("merged_linear", 0),
1199
- "up_proj": ("merged_linear", 1),
1200
- }
1201
-
1196
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1197
super().__init__()
1198
config = vllm_config.model_config.hf_config
vllm/model_executor/models/nemotron.py
-6
@@ -395,12 +395,6 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
395
"lm_head": "output_embeddings",
396
}
397
embedding_padding_modules = ["lm_head"]
398
- bitsandbytes_stacked_params_mapping = {
399
- # shard_name, weight_name, index
400
- "q_proj": ("qkv_proj", 0),
401
- "k_proj": ("qkv_proj", 1),
402
- "v_proj": ("qkv_proj", 2),
403
- }
398
399
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
400
super().__init__()
vllm/model_executor/models/opt.py
+3
-7
@@ -329,13 +329,9 @@ class OPTModel(nn.Module):
329
330
331
class OPTForCausalLM(nn.Module, SupportsPP):
332
-
333
- # BitandBytes specific attributes
334
- bitsandbytes_stacked_params_mapping = {
335
- # shard_name, weight_name, index
336
- "q_proj": ("qkv_proj", 0),
337
- "k_proj": ("qkv_proj", 1),
338
- "v_proj": ("qkv_proj", 2),
332
+ packed_modules_mapping = {
333
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
334
+ "gate_up_proj": ["gate_proj", "up_proj"]
335
}
336
337
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/phi.py
-8
@@ -279,14 +279,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
279
"fc2",
280
]
281
282
- # BitandBytes specific attributes
283
- bitsandbytes_stacked_params_mapping = {
284
- # shard_name, weight_name, index
285
- "q_proj": ("qkv_proj", 0),
286
- "k_proj": ("qkv_proj", 1),
287
- "v_proj": ("qkv_proj", 2),
288
- }
289
-
282
embedding_modules = {}
283
embedding_padding_modules = []
284
vllm/model_executor/models/phi3.py
-4
@@ -14,7 +14,3 @@ class Phi3ForCausalLM(LlamaForCausalLM):
14
"gate_up_proj",
15
],
16
}
17
-
18
- # BitandBytes specific attributes
19
- # Initialize an empty dict when there is no stacked parameter mapping.
20
- bitsandbytes_stacked_params_mapping = {}
vllm/model_executor/models/qwen.py
-7
@@ -1028,13 +1028,6 @@ class QWenLLM(QWenBaseModel):
1028
embedding_modules = {}
1029
embedding_padding_modules = []
1030
1031
- # BitandBytes specific attributes
1032
- bitsandbytes_stacked_params_mapping = {
1033
- # shard_name, weight_name, index
1034
- "w2": ("gate_up_proj", 0),
1035
- "w1": ("gate_up_proj", 1),
1036
- }
1037
-
1031
1032
class QWenVL(QWenBaseModel, SupportsMultiModal):
1033
packed_modules_mapping = {
vllm/model_executor/models/qwen2.py
-10
@@ -418,16 +418,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
418
embedding_modules = {}
419
embedding_padding_modules = []
420
421
- # BitandBytes specific attributes
422
- bitsandbytes_stacked_params_mapping = {
423
- # shard_name, weight_name, index
424
- "q_proj": ("qkv_proj", 0),
425
- "k_proj": ("qkv_proj", 1),
426
- "v_proj": ("qkv_proj", 2),
427
- "gate_proj": ("gate_up_proj", 0),
428
- "up_proj": ("gate_up_proj", 1),
429
- }
430
-
421
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
422
super().__init__()
423
config = vllm_config.model_config.hf_config
vllm/model_executor/models/qwen2_vl.py
-10
@@ -1038,16 +1038,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1038
embedding_modules = {}
1039
embedding_padding_modules = []
1040
1041
- # BitandBytes specific attributes
1042
- bitsandbytes_stacked_params_mapping = {
1043
- # shard_name, weight_name, index
1044
- "q_proj": ("qkv_proj", 0),
1045
- "k_proj": ("qkv_proj", 1),
1046
- "v_proj": ("qkv_proj", 2),
1047
- "gate_proj": ("gate_up_proj", 0),
1048
- "up_proj": ("gate_up_proj", 1),
1049
- }
1050
-
1041
# To ensure correct weight loading and mapping.
1042
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
1043
"lm_head.": "language_model.lm_head.",
vllm/model_executor/models/solar.py
-8
@@ -401,14 +401,6 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
401
"lm_head": "output_embeddings",
402
}
403
embedding_padding_modules = ["lm_head"]
404
- bitsandbytes_stacked_params_mapping = {
405
- # shard_name, weight_name, index
406
- "q_proj": ("qkv_proj", 0),
407
- "k_proj": ("qkv_proj", 1),
408
- "v_proj": ("qkv_proj", 2),
409
- "gate_proj": ("gate_up_proj", 0),
410
- "up_proj": ("gate_up_proj", 1),
411
- }
404
405
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
406
super().__init__()