A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Add Support for Multimodal Granite Models (#10291)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Alex Brooks committed
Nov 21, 2024 at 03:46 UTC
1cfde82ffd6edfca6029a7e312c848386ea322c1
6 files changed
+191
-35
vllm/model_executor/models/clip.py
+35
-12
@@ -21,7 +21,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
from vllm.multimodal.utils import (cached_get_tokenizer,
23
consecutive_placeholder_ranges,
24
- repeat_and_pad_placeholder_tokens)
24
+ repeat_and_pad_placeholder_tokens,
25
+ resolve_visual_encoder_outputs)
26
from vllm.sequence import SequenceData
27
28
from .utils import get_vit_attn_backend
@@ -389,12 +390,20 @@ class CLIPEncoder(nn.Module):
390
for layer_idx in range(num_hidden_layers)
391
])
392
392
- def forward(self, inputs_embeds: torch.Tensor):
393
-
393
+ def forward(
394
+ self, inputs_embeds: torch.Tensor, return_all_hidden_states: bool
395
+ ) -> Union[torch.Tensor, list[torch.Tensor]]:
396
+ hidden_states_pool = []
397
hidden_states = inputs_embeds
398
+
399
for encoder_layer in self.layers:
400
hidden_states = encoder_layer(hidden_states)
397
-
401
+ if return_all_hidden_states:
402
+ hidden_states_pool.append(hidden_states)
403
+ # If we have multiple feature sample layers, we return all hidden
404
+ # states in order and grab the ones we need by index.
405
+ if return_all_hidden_states:
406
+ return hidden_states_pool
407
return hidden_states
408
409
@@ -419,6 +428,7 @@ class CLIPVisionTransformer(nn.Module):
428
# NOTE: This typo of "layrnorm" is not fixed on purpose to match
429
# the original transformers code and name of the model weights.
430
self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
431
+
432
self.encoder = CLIPEncoder(
433
config=config,
434
quant_config=quant_config,
@@ -446,16 +456,26 @@ class CLIPVisionTransformer(nn.Module):
456
def forward(
457
self,
458
pixel_values: torch.Tensor,
459
+ feature_sample_layers: Optional[list[int]] = None,
460
) -> torch.Tensor:
461
462
hidden_states = self.embeddings(pixel_values)
463
hidden_states = self.pre_layrnorm(hidden_states)
453
- hidden_states = self.encoder(inputs_embeds=hidden_states)
464
455
- if self.post_layernorm is None:
456
- return hidden_states
465
+ return_all_hidden_states = feature_sample_layers is not None
466
+
467
+ # Produces either the last layer output or all of the hidden states,
468
+ # depending on if we have feature_sample_layers or not
469
+ encoder_outputs = self.encoder(
470
+ inputs_embeds=hidden_states,
471
+ return_all_hidden_states=return_all_hidden_states)
472
+
473
+ # Handle post-norm (if applicable) and stacks feature layers if needed
474
+ encoder_outputs = resolve_visual_encoder_outputs(
475
+ encoder_outputs, feature_sample_layers, self.post_layernorm,
476
+ self.config.num_hidden_layers)
477
458
- return self.post_layernorm(hidden_states)
478
+ return encoder_outputs
479
480
481
class CLIPVisionModel(nn.Module):
@@ -478,11 +498,14 @@ class CLIPVisionModel(nn.Module):
498
quant_config=quant_config,
499
num_hidden_layers_override=num_hidden_layers_override,
500
require_post_norm=require_post_norm,
481
- prefix=f"{prefix}.vision_model",
482
- )
501
+ prefix=f"{prefix}.vision_model")
502
484
- def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
485
- return self.vision_model(pixel_values)
503
+ def forward(
504
+ self,
505
+ pixel_values: torch.Tensor,
506
+ feature_sample_layers: Optional[list[int]] = None,
507
+ ) -> torch.Tensor:
508
+ return self.vision_model(pixel_values, feature_sample_layers)
509
510
@property
511
def device(self):
vllm/model_executor/models/llava.py
+37
-8
@@ -204,7 +204,41 @@ def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
204
205
class LlavaLikeConfig(Protocol):
206
vision_config: PretrainedConfig
207
- vision_feature_layer: int
207
+ vision_feature_layer: Union[int, List[int]]
208
+
209
+
210
+def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
211
+ """Determine the number of hidden layers to initialize up to in the
212
+ visual encoder.
213
+
214
+ Args:
215
+ hf_config: Model config with vision feature layer(s).
216
+ """
217
+ feature_layers = hf_config.vision_feature_layer
218
+ num_hidden_layers = hf_config.vision_config.num_hidden_layers
219
+ # If we have one feature layer, initialize up to that layer
220
+ if isinstance(feature_layers, int):
221
+ return _get_layer_index(feature_layers, num_hidden_layers)
222
+ # If we have multiple feature layers, initialize up to the deepest one
223
+ elif isinstance(feature_layers, (list, tuple)):
224
+ return max(
225
+ _get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
226
+ raise TypeError(f"vision_layer_feature type: {type(feature_layers)}"
227
+ " is not supported")
228
+
229
+
230
+def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
231
+ """Given an signed vision feature layer, get the number of hidden layers
232
+ needed to leverage it.
233
+
234
+ Args:
235
+ feature_layer_index: Index of a required layer in the visual encoder.
236
+ num_hidden_layers: The total number of hidden layers in the visual
237
+ encoder.
238
+ """
239
+ if feature_layer_index < 0:
240
+ return num_hidden_layers + feature_layer_index + 1
241
+ return feature_layer_index + 1
242
243
244
def init_vision_tower_for_llava(
@@ -216,13 +250,8 @@ def init_vision_tower_for_llava(
250
):
251
vision_config = hf_config.vision_config
252
219
- # Initialize the vision tower only up to the required feature layer
220
- vision_feature_layer = hf_config.vision_feature_layer
221
- if vision_feature_layer < 0:
222
- num_hidden_layers = hf_config.vision_config.num_hidden_layers \
223
- + vision_feature_layer + 1
224
- else:
225
- num_hidden_layers = vision_feature_layer + 1
253
+ # Initialize the vision tower only up to the deepest required feature layer
254
+ num_hidden_layers = _get_num_hidden_layers(hf_config)
255
256
if isinstance(vision_config, CLIPVisionConfig):
257
return CLIPVisionModel(
vllm/model_executor/models/llava_next.py
+18
-2
@@ -288,6 +288,21 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
288
pooler_config = vllm_config.model_config.pooler_config
289
multimodal_config = vllm_config.model_config.multimodal_config
290
291
+ vision_feature_layer = config.vision_feature_layer
292
+ # Determine the layer up to which we will initialize the vision tower
293
+ if isinstance(vision_feature_layer, int):
294
+ vision_hidden_size = config.vision_config.hidden_size
295
+ self.feature_sample_layers = None
296
+ # Used for multimodal granite models to control encoder outputs
297
+ elif isinstance(vision_feature_layer, (list, tuple)):
298
+ vision_hidden_size = config.vision_config.hidden_size * len(
299
+ vision_feature_layer)
300
+ self.feature_sample_layers = vision_feature_layer
301
+ else:
302
+ raise TypeError(
303
+ f"vision_layer_feature type: {type(vision_feature_layer)}"
304
+ " is not supported")
305
+
306
self.config = config
307
self.multimodal_config = multimodal_config
308
@@ -300,7 +315,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
315
self.image_newline = nn.Parameter(
316
torch.empty(config.text_config.hidden_size))
317
self.multi_modal_projector = LlavaMultiModalProjector(
303
- vision_hidden_size=config.vision_config.hidden_size,
318
+ vision_hidden_size=vision_hidden_size,
319
text_hidden_size=config.text_config.hidden_size,
320
projector_hidden_act=config.projector_hidden_act)
321
@@ -419,7 +434,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
434
435
# NOTE: we skip the step to select the vision feature layer since
436
# this is already done inside the vision tower
422
- image_features = vision_tower(pixel_values)
437
+ image_features = vision_tower(
438
+ pixel_values, feature_sample_layers=self.feature_sample_layers)
439
440
return self._select_image_features(
441
image_features,
vllm/model_executor/models/pixtral.py
+25
-3
@@ -33,7 +33,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
33
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
34
from vllm.multimodal.inputs import PlaceholderRange
35
from vllm.multimodal.utils import (cached_get_tokenizer,
36
- consecutive_placeholder_ranges)
36
+ consecutive_placeholder_ranges,
37
+ resolve_visual_encoder_outputs)
38
from vllm.sequence import IntermediateTensors, SequenceData
39
from vllm.transformers_utils.processor import cached_get_processor
40
from vllm.utils import is_list_of
@@ -970,9 +971,18 @@ class PixtralHFTransformer(nn.Module):
971
x: torch.Tensor,
972
attention_mask: torch.Tensor,
973
position_embeddings: torch.Tensor,
974
+ return_all_hidden_states: bool,
975
) -> torch.Tensor:
976
+ hidden_states_pool = []
977
+
978
for layer in self.layers:
979
x = layer(x, attention_mask, position_embeddings)
980
+ if return_all_hidden_states:
981
+ hidden_states_pool.append(x)
982
+ # If we have multiple feature sample layers, we return all hidden
983
+ # states in order and grab the ones we need by index.
984
+ if return_all_hidden_states:
985
+ return hidden_states_pool
986
return x
987
988
@@ -990,6 +1000,7 @@ class PixtralHFVisionModel(nn.Module):
1000
super().__init__()
1001
1002
self.config = config
1003
+
1004
self.patch_conv = nn.Conv2d(
1005
in_channels=config.num_channels,
1006
out_channels=config.hidden_size,
@@ -1024,6 +1035,7 @@ class PixtralHFVisionModel(nn.Module):
1035
def forward(
1036
self,
1037
pixel_values: List[torch.Tensor],
1038
+ feature_sample_layers: Optional[list[int]] = None,
1039
) -> torch.Tensor:
1040
"""
1041
Args:
@@ -1031,6 +1043,9 @@ class PixtralHFVisionModel(nn.Module):
1043
in pixel_values. This means it will be a list of tensors
1044
because multiple requests batched can have multiple images,
1045
each with their own shape potentially
1046
+ feature_sample_layers: Layer indices whose features should be
1047
+ concatenated and used as the visual encoder output. If none
1048
+ are provided, the last layer is used.
1049
1050
Returns:
1051
image_features: tensor of token features for
@@ -1065,8 +1080,15 @@ class PixtralHFVisionModel(nn.Module):
1080
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
1081
patch_embeds)
1082
1068
- out = self.transformer(patch_embeds, attention_mask,
1069
- position_embedding)
1083
+ return_all_hidden_states = feature_sample_layers is not None
1084
+ out = self.transformer(
1085
+ patch_embeds,
1086
+ attention_mask,
1087
+ position_embedding,
1088
+ return_all_hidden_states=return_all_hidden_states)
1089
+
1090
+ out = resolve_visual_encoder_outputs(out, feature_sample_layers, None,
1091
+ self.config.num_hidden_layers)
1092
1093
return out
1094
vllm/model_executor/models/siglip.py
+32
-10
@@ -25,7 +25,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
25
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26
from vllm.multimodal.utils import (cached_get_tokenizer,
27
consecutive_placeholder_ranges,
28
- repeat_and_pad_placeholder_tokens)
28
+ repeat_and_pad_placeholder_tokens,
29
+ resolve_visual_encoder_outputs)
30
from vllm.sequence import SequenceData
31
32
from .utils import get_vit_attn_backend
@@ -450,11 +451,19 @@ class SiglipEncoder(nn.Module):
451
def forward(
452
self,
453
inputs_embeds: torch.Tensor,
453
- ) -> torch.Tensor:
454
+ return_all_hidden_states: bool,
455
+ ) -> Union[torch.Tensor, list[torch.Tensor]]:
456
+ hidden_states_pool = []
457
hidden_states = inputs_embeds
458
+
459
for encoder_layer in self.layers:
460
hidden_states, _ = encoder_layer(hidden_states)
457
-
461
+ if return_all_hidden_states:
462
+ hidden_states_pool.append(hidden_states)
463
+ # If we have multiple feature sample layers, we return all hidden
464
+ # states in order and grab the ones we need by index.
465
+ if return_all_hidden_states:
466
+ return hidden_states_pool
467
return hidden_states
468
469
@@ -509,6 +518,7 @@ class SiglipVisionTransformer(nn.Module):
518
embed_dim = config.hidden_size
519
520
self.embeddings = SiglipVisionEmbeddings(config)
521
+
522
self.encoder = SiglipEncoder(
523
config,
524
quant_config=quant_config,
@@ -546,23 +556,33 @@ class SiglipVisionTransformer(nn.Module):
556
self,
557
pixel_values: torch.Tensor,
558
interpolate_pos_encoding: bool = True,
559
+ feature_sample_layers: Optional[list[int]] = None,
560
) -> torch.Tensor:
561
+
562
hidden_states = self.embeddings(
563
pixel_values,
564
interpolate_pos_encoding=interpolate_pos_encoding,
565
)
566
555
- encoder_outputs = self.encoder(inputs_embeds=hidden_states)
567
+ return_all_hidden_states = feature_sample_layers is not None
568
+
569
+ # Produces either the last layer output or all of the hidden states,
570
+ # depending on if we have feature_sample_layers or not
571
+ encoder_outputs = self.encoder(
572
+ inputs_embeds=hidden_states,
573
+ return_all_hidden_states=return_all_hidden_states,
574
+ )
575
557
- if self.post_layernorm is None:
558
- return encoder_outputs
576
+ # Handle post-norm (if applicable) and stacks feature layers if needed
577
+ encoder_outputs = resolve_visual_encoder_outputs(
578
+ encoder_outputs, feature_sample_layers, self.post_layernorm,
579
+ self.config.num_hidden_layers)
580
560
- last_hidden_state = self.post_layernorm(encoder_outputs)
561
- # TODO: add this back when pooled_output is used in inference
581
+ # TODO: add this back when pooled_output is used in inference.
582
# if self.use_head:
563
- # pooled_output = self.head(last_hidden_state)
583
+ # pooled_output = self.head(encoder_outputs)
584
565
- return last_hidden_state
585
+ return encoder_outputs
586
587
588
class SiglipVisionModel(nn.Module):
@@ -595,10 +615,12 @@ class SiglipVisionModel(nn.Module):
615
self,
616
pixel_values: torch.Tensor,
617
interpolate_pos_encoding: bool = False,
618
+ feature_sample_layers: Optional[list[int]] = None,
619
) -> torch.Tensor:
620
return self.vision_model(
621
pixel_values=pixel_values,
622
interpolate_pos_encoding=interpolate_pos_encoding,
623
+ feature_sample_layers=feature_sample_layers,
624
)
625
626
def load_weights(self, weights: Iterable[Tuple[str,
vllm/multimodal/utils.py
+44
@@ -6,6 +6,7 @@ from typing import Any, List, Optional, Tuple, TypeVar, Union
6
7
import numpy as np
8
import numpy.typing as npt
9
+import torch
10
from PIL import Image
11
12
import vllm.envs as envs
@@ -392,6 +393,49 @@ def encode_video_base64(frames: npt.NDArray):
393
return ",".join(base64_frames)
394
395
396
+def resolve_visual_encoder_outputs(
397
+ encoder_outputs: Union[torch.Tensor, list[torch.Tensor]],
398
+ feature_sample_layers: Optional[list[int]],
399
+ post_layer_norm: Optional[torch.nn.LayerNorm],
400
+ max_possible_layers: int,
401
+) -> torch.Tensor:
402
+ """Given the outputs a visual encoder module that may correspond to the
403
+ output of the last layer, or a list of hidden states to be stacked,
404
+ handle post normalization and resolve it into a single output tensor.
405
+
406
+ Args:
407
+ encoder_outputs: Output of encoder's last layer or all hidden states.
408
+ feature_sample_layers: Optional layer indices to grab from the encoder
409
+ outputs; if provided, encoder outputs must be a list.
410
+ post_layer_norm: Post norm to apply to the output of the encoder.
411
+ max_possible_layers: Total layers in the fully loaded visual encoder.
412
+
413
+ """
414
+ if feature_sample_layers is None:
415
+ if post_layer_norm is not None:
416
+ return post_layer_norm(encoder_outputs)
417
+ return encoder_outputs
418
+
419
+ # Get the hidden states corresponding to the layer indices.
420
+ # Negative values are relative to the full visual encoder,
421
+ # so offset them depending on how many layers were loaded.
422
+ # NOTE: this assumes that encoder_outputs contains a list
423
+ # of hidden states in the same order as the encoder layers
424
+ # that produced them.
425
+ offset = max_possible_layers - len(encoder_outputs)
426
+ hs_pool = [
427
+ encoder_outputs[layer_idx]
428
+ if layer_idx >= 0 else encoder_outputs[layer_idx + offset]
429
+ for layer_idx in feature_sample_layers
430
+ ]
431
+
432
+ # Apply post-norm on the final hidden state if we are using it
433
+ uses_last_layer = feature_sample_layers[-1] in (len(hs_pool) - 1, -1)
434
+ if post_layer_norm is not None and uses_last_layer:
435
+ hs_pool[-1] = post_layer_norm(encoder_outputs)
436
+ return torch.cat(hs_pool, dim=-1)
437
+
438
+
439
# Utilities for input processors
440
_T = TypeVar("_T", str, int)
441