@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ae1597ddf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[VLM] Minor space optimization for `ClipVisionModel` (#6436)

Roger Wang committed Jul 15, 2024 at 02:29 UTC 6ae1597ddf5ac51a6eae6b012c5399de22cb9d28
4 files changed +66 -39
vllm/model_executor/models/clip.py
+25 -21
@@ -214,22 +214,24 @@ class CLIPEncoder(nn.Module):
214
215 def __init__(self,
216 config: CLIPVisionConfig,
217 - quant_config: Optional[QuantizationConfig] = None):
217 + quant_config: Optional[QuantizationConfig] = None,
218 + num_hidden_layers_override: Optional[int] = None):
219 super().__init__()
220 self.config = config
221 +
222 + if num_hidden_layers_override is None:
223 + num_hidden_layers = config.num_hidden_layers
224 + else:
225 + num_hidden_layers = num_hidden_layers_override
226 self.layers = nn.ModuleList([
227 CLIPEncoderLayer(config=config, quant_config=quant_config)
222 - for _ in range(config.num_hidden_layers)
228 + for _ in range(num_hidden_layers)
229 ])
230
225 - def forward(self,
226 - inputs_embeds: torch.Tensor,
227 - vision_feature_layer: int = -1):
231 + def forward(self, inputs_embeds: torch.Tensor):
232
229 - # Encoder forward pass only up to the required layer
230 - num_layer = len(self.layers) + vision_feature_layer + 1
233 hidden_states = inputs_embeds
232 - for encoder_layer in self.layers[:num_layer]:
234 + for encoder_layer in self.layers:
235 hidden_states = encoder_layer(hidden_states)
236
237 return hidden_states
@@ -239,7 +241,8 @@ class CLIPVisionTransformer(nn.Module):
241
242 def __init__(self,
243 config: CLIPVisionConfig,
242 - quant_config: Optional[QuantizationConfig] = None):
244 + quant_config: Optional[QuantizationConfig] = None,
245 + num_hidden_layers_override: Optional[int] = None):
246 super().__init__()
247 self.config = config
248 embed_dim = config.hidden_size
@@ -249,18 +252,19 @@ class CLIPVisionTransformer(nn.Module):
252 # NOTE: This typo of "layrnorm" is not fixed on purpose to match
253 # the original transformers code and name of the model weights.
254 self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
252 - self.encoder = CLIPEncoder(config=config, quant_config=quant_config)
255 + self.encoder = CLIPEncoder(
256 + config=config,
257 + quant_config=quant_config,
258 + num_hidden_layers_override=num_hidden_layers_override)
259
260 def forward(
261 self,
262 pixel_values: torch.Tensor,
257 - vision_feature_layer: int = -1,
263 ) -> torch.Tensor:
264
265 hidden_states = self.embeddings(pixel_values)
266 hidden_states = self.pre_layrnorm(hidden_states)
262 - hidden_states = self.encoder(inputs_embeds=hidden_states,
263 - vision_feature_layer=vision_feature_layer)
267 + hidden_states = self.encoder(inputs_embeds=hidden_states)
268
269 return hidden_states
270
@@ -272,17 +276,17 @@ class CLIPVisionModel(nn.Module):
276
277 def __init__(self,
278 config: CLIPVisionConfig,
275 - quant_config: Optional[QuantizationConfig] = None):
279 + quant_config: Optional[QuantizationConfig] = None,
280 + num_hidden_layers_override: Optional[int] = None):
281 super().__init__()
277 - self.vision_model = CLIPVisionTransformer(config=config,
278 - quant_config=quant_config)
282 + self.vision_model = CLIPVisionTransformer(
283 + config=config,
284 + quant_config=quant_config,
285 + num_hidden_layers_override=num_hidden_layers_override)
286
280 - def forward(self,
281 - pixel_values: Optional[torch.Tensor] = None,
282 - vision_feature_layer: int = -1):
287 + def forward(self, pixel_values: Optional[torch.Tensor] = None):
288
284 - return self.vision_model(pixel_values=pixel_values,
285 - vision_feature_layer=vision_feature_layer)
289 + return self.vision_model(pixel_values=pixel_values)
290
291 @property
292 def device(self):
vllm/model_executor/models/llava.py
+12 -4
@@ -128,8 +128,17 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
128 self.config = config
129 self.multimodal_config = multimodal_config
130
131 + # Initialize the vision tower only up to the required feature layer
132 + vision_feature_layer = config.vision_feature_layer
133 + if vision_feature_layer < 0:
134 + num_hidden_layers = config.vision_config.num_hidden_layers \
135 + + vision_feature_layer + 1
136 + else:
137 + num_hidden_layers = vision_feature_layer + 1
138 +
139 # TODO: Optionally initializes this for supporting embeddings.
132 - self.vision_tower = CLIPVisionModel(config.vision_config)
140 + self.vision_tower = CLIPVisionModel(
141 + config.vision_config, num_hidden_layers_override=num_hidden_layers)
142 self.multi_modal_projector = LlavaMultiModalProjector(
143 vision_hidden_size=config.vision_config.hidden_size,
144 text_hidden_size=config.text_config.hidden_size,
@@ -193,8 +202,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
202
203 # NOTE: we skip the step to select the vision feature layer since
204 # this is already done inside the vision tower
196 - image_features = vision_tower(pixel_values,
197 - self.config.vision_feature_layer)
205 + image_features = vision_tower(pixel_values)
206
207 return self._select_image_features(
208 image_features,
@@ -333,7 +341,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
341 break
342 else:
343 use_default_weight_loading = True
336 - if use_default_weight_loading:
344 + if use_default_weight_loading and name in params_dict:
345 param = params_dict[name]
346 weight_loader = getattr(param, "weight_loader",
347 default_weight_loader)
vllm/model_executor/models/llava_next.py
+12 -4
@@ -222,8 +222,17 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
222 self.config = config
223 self.multimodal_config = multimodal_config
224
225 + # Initialize the vision tower only up to the required feature layer
226 + vision_feature_layer = config.vision_feature_layer
227 + if vision_feature_layer < 0:
228 + num_hidden_layers = config.vision_config.num_hidden_layers \
229 + + vision_feature_layer + 1
230 + else:
231 + num_hidden_layers = vision_feature_layer + 1
232 +
233 # TODO: Optionally initializes this for supporting embeddings.
226 - self.vision_tower = CLIPVisionModel(config=config.vision_config)
234 + self.vision_tower = CLIPVisionModel(
235 + config.vision_config, num_hidden_layers_override=num_hidden_layers)
236 self.multi_modal_projector = LlavaMultiModalProjector(
237 vision_hidden_size=config.vision_config.hidden_size,
238 text_hidden_size=config.text_config.hidden_size,
@@ -312,8 +321,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
321
322 # NOTE: we skip the step to select the vision feature layer since
323 # this is already done inside the vision tower
315 - image_features = vision_tower(pixel_values,
316 - self.config.vision_feature_layer)
324 + image_features = vision_tower(pixel_values)
325
326 return self._select_image_features(
327 image_features,
@@ -561,7 +569,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
569 break
570 else:
571 use_default_weight_loading = True
564 - if use_default_weight_loading:
572 + if use_default_weight_loading and name in params_dict:
573 param = params_dict[name]
574 weight_loader = getattr(param, "weight_loader",
575 default_weight_loader)
vllm/model_executor/models/phi3v.py
+17 -10
@@ -80,13 +80,11 @@ class Phi3ImageEmbeddingBase(nn.Module):
80
81 def get_img_features(self,
82 img_embeds: torch.FloatTensor) -> torch.FloatTensor:
83 - LAYER_IDX = self.layer_idx
83 TYPE_FEATURE = self.type_feature
84
85 # NOTE: we skip the step to select the vision feature layer since
86 # this is already done inside the img_processor
88 - img_feature = self.img_processor(img_embeds,
89 - vision_feature_layer=LAYER_IDX)
87 + img_feature = self.img_processor(img_embeds)
88
89 if TYPE_FEATURE == "patch":
90 patch_feature = img_feature[:, 1:]
@@ -111,7 +109,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
109 config, 'n_embd') else config.hidden_size
110
111 clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
114 - self.img_processor = CLIPVisionModel(clip_config)
112 + self.layer_idx = config.img_processor.get('layer_idx', -2)
113 +
114 + # Initialize the CLIP only up to the required feature layer
115 + if self.layer_idx < 0:
116 + num_hidden_layers = clip_config.num_hidden_layers + \
117 + self.layer_idx + 1
118 + else:
119 + num_hidden_layers = self.layer_idx + 1
120 +
121 + self.img_processor = CLIPVisionModel(
122 + clip_config, num_hidden_layers_override=num_hidden_layers)
123 image_dim_out = config.img_processor['image_dim_out']
124 self.num_img_tokens = config.img_processor['num_img_tokens']
125
@@ -142,8 +150,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
150 self.img_projection = nn.Sequential(*layers)
151
152 self.vocab_size = config.vocab_size
145 -
146 - self.layer_idx = config.img_processor.get('layer_idx', -2)
153 self.type_feature = config.img_processor.get('type_feature', 'patch')
154
155 def forward(self, input_ids: torch.LongTensor,
@@ -588,7 +594,8 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
594 # Skip loading extra bias for GPTQ models.
595 if name.endswith(".bias") and name not in params_dict:
596 continue
591 - param = params_dict[name]
592 - weight_loader = getattr(param, "weight_loader",
593 - default_weight_loader)
594 - weight_loader(param, loaded_weight)
597 + if name in params_dict:
598 + param = params_dict[name]
599 + weight_loader = getattr(param, "weight_loader",
600 + default_weight_loader)
601 + weight_loader(param, loaded_weight)