A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[VLM] Minor space optimization for `ClipVisionModel` (#6436)
Roger Wang committed
Jul 15, 2024 at 02:29 UTC
6ae1597ddf5ac51a6eae6b012c5399de22cb9d28
4 files changed
+66
-39
vllm/model_executor/models/clip.py
+25
-21
@@ -214,22 +214,24 @@ class CLIPEncoder(nn.Module):
214
215
def __init__(self,
216
config: CLIPVisionConfig,
217
- quant_config: Optional[QuantizationConfig] = None):
217
+ quant_config: Optional[QuantizationConfig] = None,
218
+ num_hidden_layers_override: Optional[int] = None):
219
super().__init__()
220
self.config = config
221
+
222
+ if num_hidden_layers_override is None:
223
+ num_hidden_layers = config.num_hidden_layers
224
+ else:
225
+ num_hidden_layers = num_hidden_layers_override
226
self.layers = nn.ModuleList([
227
CLIPEncoderLayer(config=config, quant_config=quant_config)
222
- for _ in range(config.num_hidden_layers)
228
+ for _ in range(num_hidden_layers)
229
])
230
225
- def forward(self,
226
- inputs_embeds: torch.Tensor,
227
- vision_feature_layer: int = -1):
231
+ def forward(self, inputs_embeds: torch.Tensor):
232
229
- # Encoder forward pass only up to the required layer
230
- num_layer = len(self.layers) + vision_feature_layer + 1
233
hidden_states = inputs_embeds
232
- for encoder_layer in self.layers[:num_layer]:
234
+ for encoder_layer in self.layers:
235
hidden_states = encoder_layer(hidden_states)
236
237
return hidden_states
@@ -239,7 +241,8 @@ class CLIPVisionTransformer(nn.Module):
241
242
def __init__(self,
243
config: CLIPVisionConfig,
242
- quant_config: Optional[QuantizationConfig] = None):
244
+ quant_config: Optional[QuantizationConfig] = None,
245
+ num_hidden_layers_override: Optional[int] = None):
246
super().__init__()
247
self.config = config
248
embed_dim = config.hidden_size
@@ -249,18 +252,19 @@ class CLIPVisionTransformer(nn.Module):
252
# NOTE: This typo of "layrnorm" is not fixed on purpose to match
253
# the original transformers code and name of the model weights.
254
self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
252
- self.encoder = CLIPEncoder(config=config, quant_config=quant_config)
255
+ self.encoder = CLIPEncoder(
256
+ config=config,
257
+ quant_config=quant_config,
258
+ num_hidden_layers_override=num_hidden_layers_override)
259
260
def forward(
261
self,
262
pixel_values: torch.Tensor,
257
- vision_feature_layer: int = -1,
263
) -> torch.Tensor:
264
265
hidden_states = self.embeddings(pixel_values)
266
hidden_states = self.pre_layrnorm(hidden_states)
262
- hidden_states = self.encoder(inputs_embeds=hidden_states,
263
- vision_feature_layer=vision_feature_layer)
267
+ hidden_states = self.encoder(inputs_embeds=hidden_states)
268
269
return hidden_states
270
@@ -272,17 +276,17 @@ class CLIPVisionModel(nn.Module):
276
277
def __init__(self,
278
config: CLIPVisionConfig,
275
- quant_config: Optional[QuantizationConfig] = None):
279
+ quant_config: Optional[QuantizationConfig] = None,
280
+ num_hidden_layers_override: Optional[int] = None):
281
super().__init__()
277
- self.vision_model = CLIPVisionTransformer(config=config,
278
- quant_config=quant_config)
282
+ self.vision_model = CLIPVisionTransformer(
283
+ config=config,
284
+ quant_config=quant_config,
285
+ num_hidden_layers_override=num_hidden_layers_override)
286
280
- def forward(self,
281
- pixel_values: Optional[torch.Tensor] = None,
282
- vision_feature_layer: int = -1):
287
+ def forward(self, pixel_values: Optional[torch.Tensor] = None):
288
284
- return self.vision_model(pixel_values=pixel_values,
285
- vision_feature_layer=vision_feature_layer)
289
+ return self.vision_model(pixel_values=pixel_values)
290
291
@property
292
def device(self):
vllm/model_executor/models/llava.py
+12
-4
@@ -128,8 +128,17 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
128
self.config = config
129
self.multimodal_config = multimodal_config
130
131
+ # Initialize the vision tower only up to the required feature layer
132
+ vision_feature_layer = config.vision_feature_layer
133
+ if vision_feature_layer < 0:
134
+ num_hidden_layers = config.vision_config.num_hidden_layers \
135
+ + vision_feature_layer + 1
136
+ else:
137
+ num_hidden_layers = vision_feature_layer + 1
138
+
139
# TODO: Optionally initializes this for supporting embeddings.
132
- self.vision_tower = CLIPVisionModel(config.vision_config)
140
+ self.vision_tower = CLIPVisionModel(
141
+ config.vision_config, num_hidden_layers_override=num_hidden_layers)
142
self.multi_modal_projector = LlavaMultiModalProjector(
143
vision_hidden_size=config.vision_config.hidden_size,
144
text_hidden_size=config.text_config.hidden_size,
@@ -193,8 +202,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
202
203
# NOTE: we skip the step to select the vision feature layer since
204
# this is already done inside the vision tower
196
- image_features = vision_tower(pixel_values,
197
- self.config.vision_feature_layer)
205
+ image_features = vision_tower(pixel_values)
206
207
return self._select_image_features(
208
image_features,
@@ -333,7 +341,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
341
break
342
else:
343
use_default_weight_loading = True
336
- if use_default_weight_loading:
344
+ if use_default_weight_loading and name in params_dict:
345
param = params_dict[name]
346
weight_loader = getattr(param, "weight_loader",
347
default_weight_loader)
vllm/model_executor/models/llava_next.py
+12
-4
@@ -222,8 +222,17 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
222
self.config = config
223
self.multimodal_config = multimodal_config
224
225
+ # Initialize the vision tower only up to the required feature layer
226
+ vision_feature_layer = config.vision_feature_layer
227
+ if vision_feature_layer < 0:
228
+ num_hidden_layers = config.vision_config.num_hidden_layers \
229
+ + vision_feature_layer + 1
230
+ else:
231
+ num_hidden_layers = vision_feature_layer + 1
232
+
233
# TODO: Optionally initializes this for supporting embeddings.
226
- self.vision_tower = CLIPVisionModel(config=config.vision_config)
234
+ self.vision_tower = CLIPVisionModel(
235
+ config.vision_config, num_hidden_layers_override=num_hidden_layers)
236
self.multi_modal_projector = LlavaMultiModalProjector(
237
vision_hidden_size=config.vision_config.hidden_size,
238
text_hidden_size=config.text_config.hidden_size,
@@ -312,8 +321,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
321
322
# NOTE: we skip the step to select the vision feature layer since
323
# this is already done inside the vision tower
315
- image_features = vision_tower(pixel_values,
316
- self.config.vision_feature_layer)
324
+ image_features = vision_tower(pixel_values)
325
326
return self._select_image_features(
327
image_features,
@@ -561,7 +569,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
569
break
570
else:
571
use_default_weight_loading = True
564
- if use_default_weight_loading:
572
+ if use_default_weight_loading and name in params_dict:
573
param = params_dict[name]
574
weight_loader = getattr(param, "weight_loader",
575
default_weight_loader)
vllm/model_executor/models/phi3v.py
+17
-10
@@ -80,13 +80,11 @@ class Phi3ImageEmbeddingBase(nn.Module):
80
81
def get_img_features(self,
82
img_embeds: torch.FloatTensor) -> torch.FloatTensor:
83
- LAYER_IDX = self.layer_idx
83
TYPE_FEATURE = self.type_feature
84
85
# NOTE: we skip the step to select the vision feature layer since
86
# this is already done inside the img_processor
88
- img_feature = self.img_processor(img_embeds,
89
- vision_feature_layer=LAYER_IDX)
87
+ img_feature = self.img_processor(img_embeds)
88
89
if TYPE_FEATURE == "patch":
90
patch_feature = img_feature[:, 1:]
@@ -111,7 +109,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
109
config, 'n_embd') else config.hidden_size
110
111
clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
114
- self.img_processor = CLIPVisionModel(clip_config)
112
+ self.layer_idx = config.img_processor.get('layer_idx', -2)
113
+
114
+ # Initialize the CLIP only up to the required feature layer
115
+ if self.layer_idx < 0:
116
+ num_hidden_layers = clip_config.num_hidden_layers + \
117
+ self.layer_idx + 1
118
+ else:
119
+ num_hidden_layers = self.layer_idx + 1
120
+
121
+ self.img_processor = CLIPVisionModel(
122
+ clip_config, num_hidden_layers_override=num_hidden_layers)
123
image_dim_out = config.img_processor['image_dim_out']
124
self.num_img_tokens = config.img_processor['num_img_tokens']
125
@@ -142,8 +150,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
150
self.img_projection = nn.Sequential(*layers)
151
152
self.vocab_size = config.vocab_size
145
-
146
- self.layer_idx = config.img_processor.get('layer_idx', -2)
153
self.type_feature = config.img_processor.get('type_feature', 'patch')
154
155
def forward(self, input_ids: torch.LongTensor,
@@ -588,7 +594,8 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
594
# Skip loading extra bias for GPTQ models.
595
if name.endswith(".bias") and name not in params_dict:
596
continue
591
- param = params_dict[name]
592
- weight_loader = getattr(param, "weight_loader",
593
- default_weight_loader)
594
- weight_loader(param, loaded_weight)
597
+ if name in params_dict:
598
+ param = params_dict[name]
599
+ weight_loader = getattr(param, "weight_loader",
600
+ default_weight_loader)
601
+ weight_loader(param, loaded_weight)