@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c49f0407ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Fix MiniCPMV and Mllama BNB bug (#9917)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 4, 2024 at 11:36 UTC c49f0407ba60bfee538892a09561c1fe7484adf8
4 files changed +145 -65
vllm/model_executor/layers/resampler.py
+28 -21
@@ -41,6 +41,7 @@ from torch import nn
41 from torch.nn.init import trunc_normal_
42
43 from vllm.model_executor.layers.linear import ReplicatedLinear
44 +from vllm.model_executor.layers.quantization import QuantizationConfig
45
46 DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
47
@@ -154,15 +155,15 @@ class BaseResampler(nn.Module):
155 A tensor with the shape of (grid_size**2, embed_dim)
156 """
157
157 - def __init__(
158 - self,
159 - num_queries: int,
160 - embed_dim: int,
161 - num_heads: int,
162 - kv_dim: Optional[int] = None,
163 - norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
164 - do_post_projection: bool = True,
165 - ) -> None:
158 + def __init__(self,
159 + num_queries: int,
160 + embed_dim: int,
161 + num_heads: int,
162 + kv_dim: Optional[int] = None,
163 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
164 + do_post_projection: bool = True,
165 + quant_config: Optional[QuantizationConfig] = None,
166 + prefix: str = "") -> None:
167 super().__init__()
168
169 self.num_queries = num_queries
@@ -172,7 +173,11 @@ class BaseResampler(nn.Module):
173 self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
174 trunc_normal_(self.query, std=0.02)
175 if kv_dim is not None and kv_dim != embed_dim:
175 - self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
176 + self.kv_proj = ReplicatedLinear(kv_dim,
177 + embed_dim,
178 + bias=False,
179 + quant_config=quant_config,
180 + prefix=prefix)
181 else:
182 # Maintain the same return value with ReplicatedLinear.forward
183 self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
@@ -209,22 +214,24 @@ class Resampler2(BaseResampler):
214 present in minicpmv2.0, but not qwen-vl.
215 """
216
212 - def __init__(
213 - self,
214 - grid_size: int,
215 - embed_dim: int,
216 - num_heads: int,
217 - kv_dim: Optional[int] = None,
218 - norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
219 - adaptive: bool = False,
220 - do_post_projection: bool = True,
221 - ) -> None:
217 + def __init__(self,
218 + grid_size: int,
219 + embed_dim: int,
220 + num_heads: int,
221 + kv_dim: Optional[int] = None,
222 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
223 + adaptive: bool = False,
224 + do_post_projection: bool = True,
225 + quant_config: Optional[QuantizationConfig] = None,
226 + prefix: str = "") -> None:
227 super().__init__(grid_size**2,
228 embed_dim,
229 num_heads,
230 kv_dim,
231 norm_layer,
227 - do_post_projection=do_post_projection)
232 + do_post_projection=do_post_projection,
233 + quant_config=quant_config,
234 + prefix=prefix)
235
236 self.adaptive = adaptive
237 pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
vllm/model_executor/model_loader/loader.py
+28 -6
@@ -28,6 +28,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.envs import VLLM_USE_MODELSCOPE
30 from vllm.logger import init_logger
31 +from vllm.model_executor.layers.linear import ReplicatedLinear
32 from vllm.model_executor.layers.quantization.base_config import (
33 QuantizationConfig)
34 from vllm.model_executor.model_loader.tensorizer import (
@@ -771,6 +772,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
772 with open(config_file_path, "r") as f:
773 config = json.load(f)
774 self.target_modules = config["target_modules"]
775 + # Save the module names without sharding.
776 + self.unsharded_weights_modules: List[str] = []
777
778 def _get_config_file(self, qlora_adapter: str) -> str:
779 is_local = os.path.isdir(qlora_adapter)
@@ -990,16 +993,21 @@ class BitsAndBytesModelLoader(BaseModelLoader):
993 if any(target_module in weight_name for target_module in
994 self.target_modules) and weight_name.endswith(".weight"):
995 weight_name = weight_name.replace(".weight", ".qweight")
993 -
994 - if any(module in weight_name
995 - for module in self.column_parallel_weights_modules):
996 + # Without sharding
997 + if any(
998 + weight_name.startswith(module)
999 + for module in self.unsharded_weights_modules):
1000 + weight_sub_tensor = weight_tensor
1001 + # Shard by column
1002 + elif any(module in weight_name
1003 + for module in self.column_parallel_weights_modules):
1004
1005 total_size = weight_tensor.size(-1)
1006 start_index = total_size // tp_size * tp_rank
1007 end_index = total_size // tp_size * (tp_rank + 1)
1008 weight_sub_tensor = weight_tensor[...,
1009 start_index:end_index]
1002 -
1010 + # Shard by row
1011 else:
1012 total_size = weight_tensor.size(0)
1013 start_index = total_size // tp_size * tp_rank
@@ -1053,7 +1061,15 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1061 model.column_parallel_weights_modules
1062 else:
1063 self.column_parallel_weights_modules = []
1056 -
1064 + # Some modules like `ReplicatedLinear` should not have their weights
1065 + # sharded. The reason for implementing it this way is to avoid new
1066 + # static variable in the model implementation.
1067 + # TODO: Can we reduce the static variables needed for BNB based on
1068 + # model information?
1069 + self.unsharded_weights_modules = [
1070 + name for name, module in model.named_modules()
1071 + if isinstance(module, (ReplicatedLinear, ))
1072 + ]
1073 self.model_type = type(model).__name__
1074
1075 logger.info("Loading weights with BitsAndBytes quantization. "
@@ -1100,7 +1116,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1116 for shard_name, (
1117 weight_name, index
1118 ) in model.bitsandbytes_stacked_params_mapping.items():
1103 - if shard_name in quant_param_name:
1119 +
1120 + shard_pos = quant_param_name.find(shard_name)
1121 + # Some models, such as MiniCPM V2.5/2.6, contain both
1122 + # module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
1123 + # from being incorrectly identified as being present in
1124 + # 'vpm.encoder.layers.0.self_attn.qkv_proj.qweight
1125 + if shard_pos > 0 and quant_param_name[shard_pos - 1] == ".":
1126 shard_index = index
1127 quant_param_name = quant_param_name.replace(
1128 shard_name, weight_name)
vllm/model_executor/models/minicpmv.py
+83 -37
@@ -131,16 +131,22 @@ DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
131
132 class Resampler2_5(BaseResampler):
133
134 - def __init__(
135 - self,
136 - num_queries: int,
137 - embed_dim: int,
138 - num_heads: int,
139 - kv_dim: Optional[int] = None,
140 - norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
141 - max_size: Tuple[int, int] = (70, 70),
142 - ) -> None:
143 - super().__init__(num_queries, embed_dim, num_heads, kv_dim, norm_layer)
134 + def __init__(self,
135 + num_queries: int,
136 + embed_dim: int,
137 + num_heads: int,
138 + kv_dim: Optional[int] = None,
139 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
140 + max_size: Tuple[int, int] = (70, 70),
141 + quant_config: Optional[QuantizationConfig] = None,
142 + prefix: str = "") -> None:
143 + super().__init__(num_queries,
144 + embed_dim,
145 + num_heads,
146 + kv_dim,
147 + norm_layer,
148 + quant_config=quant_config,
149 + prefix=prefix)
150
151 self.max_size = max_size
152 self._set_2d_pos_cache(self.max_size)
@@ -404,7 +410,10 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
410 self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
411 self.vpm.embeddings.embed_dim)
412 self.embed_dim = self.config.hidden_size
407 - self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
413 + self.resampler = self.init_resampler(self.embed_dim,
414 + self.vision_dim,
415 + quant_config=quant_config,
416 + prefix="resampler")
417 self.resampler.to(device="cuda", dtype=param_dtype)
418 # TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
419 self.lm_head = ParallelLMHead(config.vocab_size,
@@ -666,7 +675,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
675 ) -> nn.Module:
676 raise NotImplementedError
677
669 - def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
678 + def init_resampler(self,
679 + embed_dim: int,
680 + vision_dim: int,
681 + quant_config: Optional[QuantizationConfig] = None,
682 + prefix: str = "") -> nn.Module:
683 raise NotImplementedError
684
685 def get_vision_embedding(
@@ -743,16 +756,21 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
756 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
757 return self.model.embed_tokens(input_ids)
758
746 - def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
759 + def init_resampler(self,
760 + embed_dim: int,
761 + vision_dim: int,
762 + quant_config: Optional[QuantizationConfig] = None,
763 + prefix: str = "") -> nn.Module:
764 with set_default_torch_dtype(torch.float16):
748 - resampler = Resampler2(
749 - embed_dim=embed_dim,
750 - num_heads=embed_dim // 128,
751 - grid_size=int(math.sqrt(self.config.query_num)),
752 - kv_dim=vision_dim,
753 - adaptive=False,
754 - do_post_projection=True,
755 - )
765 + resampler = Resampler2(embed_dim=embed_dim,
766 + num_heads=embed_dim // 128,
767 + grid_size=int(
768 + math.sqrt(self.config.query_num)),
769 + kv_dim=vision_dim,
770 + adaptive=False,
771 + do_post_projection=True,
772 + quant_config=quant_config,
773 + prefix=prefix)
774
775 return resampler
776
@@ -825,9 +843,21 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
843 ".k_proj.",
844 ".v_proj.",
845 ".o_proj.",
846 + # vision encoder
847 + ".fc1.",
848 + ".fc2.",
849 + # Currently, vllm does not support BNB quantization for the `out_proj`
850 + # of the resampler, so it's necessary to distinguish between the
851 + # vision encoder and the resampler's out_proj. The same applies to
852 + # MiniCPMV2_6.
853 + ".self_attn.out_proj.", # vision encoder out_proj
854 + # resampler
855 + ".kv_proj.",
856 ]
857 # in TP, these weights are partitioned along the column dimension (dim=-1)
830 - column_parallel_weights_modules = [".down_proj.", ".o_proj."]
858 + column_parallel_weights_modules = [
859 + ".down_proj.", ".o_proj.", ".self_attn.out_proj.", ".fc2."
860 + ]
861 bitsandbytes_stacked_params_mapping = {
862 # shard_name, weight_name, index
863 "q_proj": ("qkv_proj", 0),
@@ -877,14 +907,18 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
907 model.encoder.layers = model.encoder.layers[:-1]
908 return model
909
880 - def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
910 + def init_resampler(self,
911 + embed_dim: int,
912 + vision_dim: int,
913 + quant_config: Optional[QuantizationConfig] = None,
914 + prefix: str = "") -> nn.Module:
915 with set_default_torch_dtype(torch.float16):
882 - resampler = Resampler2_5(
883 - num_queries=self.config.query_num,
884 - embed_dim=embed_dim,
885 - num_heads=embed_dim // 128,
886 - kv_dim=vision_dim,
887 - )
916 + resampler = Resampler2_5(num_queries=self.config.query_num,
917 + embed_dim=embed_dim,
918 + num_heads=embed_dim // 128,
919 + kv_dim=vision_dim,
920 + quant_config=quant_config,
921 + prefix=prefix)
922 return resampler
923
924 def get_vision_embedding(
@@ -967,9 +1001,17 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1001 ".k_proj.",
1002 ".v_proj.",
1003 ".o_proj.",
1004 + # vision encoder
1005 + ".fc1.",
1006 + ".fc2.",
1007 + ".self_attn.out_proj.",
1008 + # resampler
1009 + ".kv_proj.",
1010 ]
1011 # in TP, these weights are partitioned along the column dimension (dim=-1)
972 - column_parallel_weights_modules = [".down_proj.", ".o_proj."]
1012 + column_parallel_weights_modules = [
1013 + ".down_proj.", ".o_proj.", ".self_attn.out_proj.", ".fc2."
1014 + ]
1015 bitsandbytes_stacked_params_mapping = {
1016 # shard_name, weight_name, index
1017 "q_proj": ("qkv_proj", 0),
@@ -1019,15 +1061,19 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1061 model.encoder.layers = model.encoder.layers[:-1]
1062 return model
1063
1022 - def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
1064 + def init_resampler(self,
1065 + embed_dim: int,
1066 + vision_dim: int,
1067 + quant_config: Optional[QuantizationConfig] = None,
1068 + prefix: str = "") -> nn.Module:
1069 with set_default_torch_dtype(torch.float16):
1070 # The resampler in 2.6 remains consistent with the one in 2.5.
1025 - resampler = Resampler2_5(
1026 - num_queries=self.config.query_num,
1027 - embed_dim=embed_dim,
1028 - num_heads=embed_dim // 128,
1029 - kv_dim=vision_dim,
1030 - )
1071 + resampler = Resampler2_5(num_queries=self.config.query_num,
1072 + embed_dim=embed_dim,
1073 + num_heads=embed_dim // 128,
1074 + kv_dim=vision_dim,
1075 + quant_config=quant_config,
1076 + prefix=prefix)
1077 return resampler
1078
1079 def get_vision_embedding(
vllm/model_executor/models/mllama.py
+6 -1
@@ -1056,9 +1056,14 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1056 ".k_proj.",
1057 ".v_proj.",
1058 ".o_proj.",
1059 + ".fc1.",
1060 + ".fc2.",
1061 + # The `multi_modal_projector` is at the top level of the model,
1062 + # so we can't add a dot in front of it.
1063 + "multi_modal_projector."
1064 ]
1065 # in TP, these weights are partitioned along the column dimension (dim=-1)
1061 - column_parallel_weights_modules = [".down_proj.", ".o_proj."]
1066 + column_parallel_weights_modules = [".down_proj.", ".o_proj.", ".fc2."]
1067 bitsandbytes_stacked_params_mapping = {
1068 # shard_name, weight_name, index
1069 "q_proj": ("qkv_proj", 0),