@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a3a3ee4e6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc] Merge bitsandbytes_stacked_params_mapping and packed_modules_mapping (#11924)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Jan 15, 2025 at 07:49 UTC a3a3ee4e6febe8c270fdec0765c844186a728079
24 files changed +49 -200
vllm/model_executor/model_loader/loader.py
+11 -16
@@ -39,7 +39,8 @@ from vllm.model_executor.layers.quantization.base_config import (
39 from vllm.model_executor.model_loader.tensorizer import (
40 TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
41 serialize_vllm_model, tensorizer_weights_iterator)
42 -from vllm.model_executor.model_loader.utils import (get_model_architecture,
42 +from vllm.model_executor.model_loader.utils import (ParamMapping,
43 + get_model_architecture,
44 set_default_torch_dtype)
45 from vllm.model_executor.model_loader.weight_utils import (
46 download_safetensors_index_file_from_hf, download_weights_from_hf,
@@ -983,21 +984,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
984
985 def _get_bnb_target_modules(self, model: nn.Module) -> None:
986
986 - # TODO: Maybe we can replace bitsandbytes_stacked_params_mapping with
987 - # packed_modules_mapping.
988 - inverse_stacked_mapping: Dict[str, List[str]] = {}
989 - for orig, (
990 - packed,
991 - idx,
992 - ) in model.bitsandbytes_stacked_params_mapping.items():
993 - if packed not in inverse_stacked_mapping:
994 - inverse_stacked_mapping[packed] = []
995 - inverse_stacked_mapping[packed].insert(idx, orig)
996 -
987 for name, module in model.named_modules():
988 if isinstance(module, (LinearBase, )):
989 last_name = name.split(".")[-1]
1000 - if sub_modules := inverse_stacked_mapping.get(last_name, []):
990 + if sub_modules := self.modules_mapping.packed_mapping.get(
991 + last_name, []):
992 # Map vllm's names to transformers's names.
993 for sub_name in sub_modules:
994 self.target_modules.append(
@@ -1018,15 +1009,19 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1009 "The required method 'load_weights' is not defined in class"
1010 f" {type(model).__name__}.")
1011
1021 - if not hasattr(model, "bitsandbytes_stacked_params_mapping"):
1012 + if not hasattr(model, "packed_modules_mapping"):
1013 raise AttributeError(
1014 f"Model {type(model).__name__} does not support BitsAndBytes "
1024 - "quantization yet.")
1015 + "quantization yet. No 'packed_modules_mapping' found.")
1016 +
1017 + self.modules_mapping = ParamMapping(
1018 + copy.deepcopy(model.packed_modules_mapping))
1019
1020 # For some models like Molmo, we need to use hf_to_vllm_mapper
1021 # to ensure correct loading of weights.
1022 if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):
1023 self.weight_mapper = lambda name: hf_to_vllm_mapper._map_name(name)
1024 +
1025 # Modules whose weights might have fused on disk
1026 # we need their output_sizes to make shard in flight correctly with TP
1027 self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
@@ -1109,7 +1104,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1104 for shard_name, (
1105 weight_name,
1106 index,
1112 - ) in model.bitsandbytes_stacked_params_mapping.items():
1107 + ) in self.modules_mapping.inverse_packed_mapping.items():
1108 shard_pos = quant_param_name.find(shard_name)
1109 # Some models, such as MiniCPM V2.5/2.6, contain both
1110 # module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
vllm/model_executor/model_loader/utils.py
+25 -1
@@ -1,6 +1,7 @@
1 """Utilities for selecting and loading models."""
2 import contextlib
3 -from typing import Tuple, Type
3 +from dataclasses import dataclass, field
4 +from typing import Dict, List, Tuple, Type
5
6 import torch
7 from torch import nn
@@ -49,3 +50,26 @@ def get_model_architecture(
50
51 def get_architecture_class_name(model_config: ModelConfig) -> str:
52 return get_model_architecture(model_config)[1]
53 +
54 +
55 +@dataclass
56 +class ParamMapping:
57 + """
58 + A class to handle parameter mapping for model weight loading.
59 + It creates a bidirectional mapping between packed parameters and their
60 + constituent parts.
61 + """
62 + packed_mapping: Dict[str, List[str]]
63 + inverse_packed_mapping: Dict[str, Tuple[str,
64 + int]] = field(default_factory=dict)
65 +
66 + def __post_init__(self):
67 + for packed_name, sub_params in self.packed_mapping.items():
68 + # Skip self-contained cases (e.g., {"W_pack": ["W_pack"]})
69 + if len(sub_params) == 1 and sub_params[0] == packed_name:
70 + continue
71 + for index, param_name in enumerate(sub_params):
72 + self.inverse_packed_mapping[param_name] = (
73 + packed_name,
74 + index,
75 + )
vllm/model_executor/models/baichuan.py
-7
@@ -350,13 +350,6 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350 embedding_modules = {}
351 embedding_padding_modules = []
352
353 - # BitandBytes specific attributes
354 - bitsandbytes_stacked_params_mapping = {
355 - # shard_name, weight_name, index
356 - "gate_proj": ("gate_up_proj", 0),
357 - "up_proj": ("gate_up_proj", 1),
358 - }
359 -
353 def __init__(
354 self,
355 *,
vllm/model_executor/models/exaone.py
-8
@@ -430,14 +430,6 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
430 "lm_head": "output_embeddings",
431 }
432 embedding_padding_modules = ["lm_head"]
433 - bitsandbytes_stacked_params_mapping = {
434 - # shard_name, weight_name, index
435 - "q_proj": ("qkv_proj", 0),
436 - "k_proj": ("qkv_proj", 1),
437 - "v_proj": ("qkv_proj", 2),
438 - "c_fc_0": ("gate_up_proj", 0),
439 - "c_fc_1": ("gate_up_proj", 1),
440 - }
433
434 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
435 super().__init__()
vllm/model_executor/models/falcon.py
+3 -3
@@ -409,9 +409,9 @@ class FalconModel(nn.Module):
409
410
411 class FalconForCausalLM(nn.Module, SupportsPP):
412 -
413 - # BitandBytes specific attributes
414 - bitsandbytes_stacked_params_mapping = {}
412 + packed_modules_mapping = {
413 + "query_key_value": ["query_key_value"],
414 + }
415
416 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
417 super().__init__()
vllm/model_executor/models/gemma.py
-9
@@ -349,15 +349,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
349 "gate_up_proj",
350 "down_proj",
351 ]
352 - # BitandBytes specific attributes
353 - bitsandbytes_stacked_params_mapping = {
354 - # shard_name, weight_name, index
355 - "q_proj": ("qkv_proj", 0),
356 - "k_proj": ("qkv_proj", 1),
357 - "v_proj": ("qkv_proj", 2),
358 - "gate_proj": ("gate_up_proj", 0),
359 - "up_proj": ("gate_up_proj", 1),
360 - }
352
353 # Gemma does not apply LoRA to the embedding layer.
354 embedding_modules = {}
vllm/model_executor/models/gemma2.py
-10
@@ -399,16 +399,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
399 embedding_modules = {}
400 embedding_padding_modules = []
401
402 - # BitandBytes specific attributes
403 - bitsandbytes_stacked_params_mapping = {
404 - # shard_name, weight_name, index
405 - "q_proj": ("qkv_proj", 0),
406 - "k_proj": ("qkv_proj", 1),
407 - "v_proj": ("qkv_proj", 2),
408 - "gate_proj": ("gate_up_proj", 0),
409 - "up_proj": ("gate_up_proj", 1),
410 - }
411 -
402 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
403 config = vllm_config.model_config.hf_config
404 quant_config = vllm_config.quant_config
vllm/model_executor/models/granite.py
-8
@@ -362,14 +362,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
362 "lm_head": "output_embeddings",
363 }
364 embedding_padding_modules = ["lm_head"]
365 - bitsandbytes_stacked_params_mapping = {
366 - # shard_name, weight_name, index
367 - "q_proj": ("qkv_proj", 0),
368 - "k_proj": ("qkv_proj", 1),
369 - "v_proj": ("qkv_proj", 2),
370 - "gate_proj": ("gate_up_proj", 0),
371 - "up_proj": ("gate_up_proj", 1),
372 - }
365
366 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
367 super().__init__()
vllm/model_executor/models/idefics3.py
-10
@@ -662,16 +662,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
662 "down_proj",
663 ]
664
665 - # BitandBytes specific attributes
666 - bitsandbytes_stacked_params_mapping = {
667 - # shard_name, weight_name, index
668 - "q_proj": ("qkv_proj", 0),
669 - "k_proj": ("qkv_proj", 1),
670 - "v_proj": ("qkv_proj", 2),
671 - "gate_proj": ("gate_up_proj", 0),
672 - "up_proj": ("gate_up_proj", 1),
673 - }
674 -
665 embedding_modules = {}
666 embedding_padding_modules = []
667
vllm/model_executor/models/llama.py
-10
@@ -478,16 +478,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
478 }
479 embedding_padding_modules = ["lm_head"]
480
481 - # BitandBytes specific attributes
482 - bitsandbytes_stacked_params_mapping = {
483 - # shard_name, weight_name, index
484 - "q_proj": ("qkv_proj", 0),
485 - "k_proj": ("qkv_proj", 1),
486 - "v_proj": ("qkv_proj", 2),
487 - "gate_proj": ("gate_up_proj", 0),
488 - "up_proj": ("gate_up_proj", 1),
489 - }
490 -
481 # Mistral/Llama models can also be loaded with --load-format mistral
482 # from consolidated.safetensors checkpoints
483 mistral_mapping = {
vllm/model_executor/models/llava.py
+4 -8
@@ -463,14 +463,10 @@ def init_vision_tower_for_llava(
463 info=_build_llava_or_pixtral_hf_info,
464 dummy_inputs=LlavaDummyInputsBuilder)
465 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
466 - # BitandBytes specific attributes
467 - bitsandbytes_stacked_params_mapping = {
468 - # shard_name, weight_name, index
469 - "q_proj": ("qkv_proj", 0),
470 - "k_proj": ("qkv_proj", 1),
471 - "v_proj": ("qkv_proj", 2),
472 - "gate_proj": ("gate_up_proj", 0),
473 - "up_proj": ("gate_up_proj", 1),
466 +
467 + packed_modules_mapping = {
468 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
469 + "gate_up_proj": ["gate_proj", "up_proj"]
470 }
471
472 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
vllm/model_executor/models/minicpm.py
-10
@@ -534,16 +534,6 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
534 }
535 embedding_padding_modules = ["lm_head"]
536
537 - # BitandBytes specific attributes
538 - bitsandbytes_stacked_params_mapping = {
539 - # shard_name, weight_name, index
540 - "q_proj": ("qkv_proj", 0),
541 - "k_proj": ("qkv_proj", 1),
542 - "v_proj": ("qkv_proj", 2),
543 - "gate_proj": ("gate_up_proj", 0),
544 - "up_proj": ("gate_up_proj", 1),
545 - }
546 -
537 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
538 super().__init__()
539 config = vllm_config.model_config.hf_config
vllm/model_executor/models/minicpm3.py
-6
@@ -241,11 +241,5 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
241 # `embedding_modules` and `embedding_padding_modules`
242 # are inherited from MiniCPMForCausalLM
243
244 - bitsandbytes_stacked_params_mapping = {
245 - # shard_name, weight_name, index
246 - "gate_proj": ("gate_up_proj", 0),
247 - "up_proj": ("gate_up_proj", 1),
248 - }
249 -
244 def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
245 return MiniCPM3Model(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/minicpmv.py
-20
@@ -761,16 +761,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
761 "kv_proj",
762 ]
763
764 - # BitandBytes specific attributes
765 - bitsandbytes_stacked_params_mapping = {
766 - # shard_name, weight_name, index
767 - "q_proj": ("qkv_proj", 0),
768 - "k_proj": ("qkv_proj", 1),
769 - "v_proj": ("qkv_proj", 2),
770 - "gate_proj": ("gate_up_proj", 0),
771 - "up_proj": ("gate_up_proj", 1),
772 - }
773 -
764 embedding_modules = {}
765 embedding_padding_modules = []
766
@@ -881,16 +871,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
871 "kv_proj",
872 ]
873
884 - # BitandBytes specific attributes
885 - bitsandbytes_stacked_params_mapping = {
886 - # shard_name, weight_name, index
887 - "q_proj": ("qkv_proj", 0),
888 - "k_proj": ("qkv_proj", 1),
889 - "v_proj": ("qkv_proj", 2),
890 - "gate_proj": ("gate_up_proj", 0),
891 - "up_proj": ("gate_up_proj", 1),
892 - }
893 -
874 embedding_modules = {}
875 embedding_padding_modules = []
876
vllm/model_executor/models/mllama.py
+3 -8
@@ -1107,14 +1107,9 @@ class MllamaForCausalLM(nn.Module):
1107 @INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
1108 @INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1109 class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1110 - # BitandBytes specific attributes
1111 - bitsandbytes_stacked_params_mapping = {
1112 - # shard_name, weight_name, index
1113 - "q_proj": ("qkv_proj", 0),
1114 - "k_proj": ("qkv_proj", 1),
1115 - "v_proj": ("qkv_proj", 2),
1116 - "gate_proj": ("gate_up_proj", 0),
1117 - "up_proj": ("gate_up_proj", 1),
1110 + packed_modules_mapping = {
1111 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
1112 + "gate_up_proj": ["gate_proj", "up_proj"]
1113 }
1114
1115 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/molmo.py
-6
@@ -1193,12 +1193,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1193 embedding_modules = {}
1194 embedding_padding_modules = []
1195
1196 - # BitandBytes specific attributes
1197 - bitsandbytes_stacked_params_mapping = {
1198 - "gate_proj": ("merged_linear", 0),
1199 - "up_proj": ("merged_linear", 1),
1200 - }
1201 -
1196 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1197 super().__init__()
1198 config = vllm_config.model_config.hf_config
vllm/model_executor/models/nemotron.py
-6
@@ -395,12 +395,6 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
395 "lm_head": "output_embeddings",
396 }
397 embedding_padding_modules = ["lm_head"]
398 - bitsandbytes_stacked_params_mapping = {
399 - # shard_name, weight_name, index
400 - "q_proj": ("qkv_proj", 0),
401 - "k_proj": ("qkv_proj", 1),
402 - "v_proj": ("qkv_proj", 2),
403 - }
398
399 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
400 super().__init__()
vllm/model_executor/models/opt.py
+3 -7
@@ -329,13 +329,9 @@ class OPTModel(nn.Module):
329
330
331 class OPTForCausalLM(nn.Module, SupportsPP):
332 -
333 - # BitandBytes specific attributes
334 - bitsandbytes_stacked_params_mapping = {
335 - # shard_name, weight_name, index
336 - "q_proj": ("qkv_proj", 0),
337 - "k_proj": ("qkv_proj", 1),
338 - "v_proj": ("qkv_proj", 2),
332 + packed_modules_mapping = {
333 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
334 + "gate_up_proj": ["gate_proj", "up_proj"]
335 }
336
337 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/phi.py
-8
@@ -279,14 +279,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
279 "fc2",
280 ]
281
282 - # BitandBytes specific attributes
283 - bitsandbytes_stacked_params_mapping = {
284 - # shard_name, weight_name, index
285 - "q_proj": ("qkv_proj", 0),
286 - "k_proj": ("qkv_proj", 1),
287 - "v_proj": ("qkv_proj", 2),
288 - }
289 -
282 embedding_modules = {}
283 embedding_padding_modules = []
284
vllm/model_executor/models/phi3.py
-4
@@ -14,7 +14,3 @@ class Phi3ForCausalLM(LlamaForCausalLM):
14 "gate_up_proj",
15 ],
16 }
17 -
18 - # BitandBytes specific attributes
19 - # Initialize an empty dict when there is no stacked parameter mapping.
20 - bitsandbytes_stacked_params_mapping = {}
vllm/model_executor/models/qwen.py
-7
@@ -1028,13 +1028,6 @@ class QWenLLM(QWenBaseModel):
1028 embedding_modules = {}
1029 embedding_padding_modules = []
1030
1031 - # BitandBytes specific attributes
1032 - bitsandbytes_stacked_params_mapping = {
1033 - # shard_name, weight_name, index
1034 - "w2": ("gate_up_proj", 0),
1035 - "w1": ("gate_up_proj", 1),
1036 - }
1037 -
1031
1032 class QWenVL(QWenBaseModel, SupportsMultiModal):
1033 packed_modules_mapping = {
vllm/model_executor/models/qwen2.py
-10
@@ -418,16 +418,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
418 embedding_modules = {}
419 embedding_padding_modules = []
420
421 - # BitandBytes specific attributes
422 - bitsandbytes_stacked_params_mapping = {
423 - # shard_name, weight_name, index
424 - "q_proj": ("qkv_proj", 0),
425 - "k_proj": ("qkv_proj", 1),
426 - "v_proj": ("qkv_proj", 2),
427 - "gate_proj": ("gate_up_proj", 0),
428 - "up_proj": ("gate_up_proj", 1),
429 - }
430 -
421 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
422 super().__init__()
423 config = vllm_config.model_config.hf_config
vllm/model_executor/models/qwen2_vl.py
-10
@@ -1038,16 +1038,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1038 embedding_modules = {}
1039 embedding_padding_modules = []
1040
1041 - # BitandBytes specific attributes
1042 - bitsandbytes_stacked_params_mapping = {
1043 - # shard_name, weight_name, index
1044 - "q_proj": ("qkv_proj", 0),
1045 - "k_proj": ("qkv_proj", 1),
1046 - "v_proj": ("qkv_proj", 2),
1047 - "gate_proj": ("gate_up_proj", 0),
1048 - "up_proj": ("gate_up_proj", 1),
1049 - }
1050 -
1041 # To ensure correct weight loading and mapping.
1042 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
1043 "lm_head.": "language_model.lm_head.",
vllm/model_executor/models/solar.py
-8
@@ -401,14 +401,6 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
401 "lm_head": "output_embeddings",
402 }
403 embedding_padding_modules = ["lm_head"]
404 - bitsandbytes_stacked_params_mapping = {
405 - # shard_name, weight_name, index
406 - "q_proj": ("qkv_proj", 0),
407 - "k_proj": ("qkv_proj", 1),
408 - "v_proj": ("qkv_proj", 2),
409 - "gate_proj": ("gate_up_proj", 0),
410 - "up_proj": ("gate_up_proj", 1),
411 - }
404
405 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
406 super().__init__()