@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad137cd111
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] Port over CLIPVisionModel for VLMs (#5591)

Roger Wang committed Jun 20, 2024 at 04:52 UTC ad137cd1112ab9b17ac36fc123fc7806a1d7473d
9 files changed +269 -21
csrc/activation_kernels.cu
+12
@@ -135,6 +135,12 @@ __device__ __forceinline__ T gelu_fast_kernel(const T& x) {
135 return ((T)0.5) * x * (((T)1.0) + t);
136 }
137
138 +template <typename T>
139 +__device__ __forceinline__ T gelu_quick_kernel(const T& x) {
140 + // x * sigmoid(1.702 * x)
141 + return (T)(((float)x) / (1.0f + expf(-1.702f * (float)x)));
142 +}
143 +
144 } // namespace vllm
145
146 void gelu_new(torch::Tensor& out, // [..., d]
@@ -148,3 +154,9 @@ void gelu_fast(torch::Tensor& out, // [..., d]
154 {
155 LAUNCH_ACTIVATION_KERNEL(vllm::gelu_fast_kernel);
156 }
157 +
158 +void gelu_quick(torch::Tensor& out, // [..., d]
159 + torch::Tensor& input) // [..., d]
160 +{
161 + LAUNCH_ACTIVATION_KERNEL(vllm::gelu_quick_kernel);
162 +}
csrc/ops.h
+2
@@ -49,6 +49,8 @@ void gelu_new(torch::Tensor& out, torch::Tensor& input);
49
50 void gelu_fast(torch::Tensor& out, torch::Tensor& input);
51
52 +void gelu_quick(torch::Tensor& out, torch::Tensor& input);
53 +
54 #ifndef USE_ROCM
55 torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
56 const torch::Tensor& codebooks,
csrc/torch_bindings.cpp
+4
@@ -68,6 +68,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
68 ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
69 ops.impl("gelu_fast", torch::kCUDA, &gelu_fast);
70
71 + // Quick GELU implementation.
72 + ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
73 + ops.impl("gelu_quick", torch::kCUDA, &gelu_quick);
74 +
75 // Layernorm
76 // Apply Root Mean Square (RMS) Normalization to the input tensor.
77 ops.def(
vllm/_custom_ops.py
+4
@@ -66,6 +66,10 @@ def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
66 torch.ops._C.gelu_new(out, x)
67
68
69 +def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
70 + torch.ops._C.gelu_quick(out, x)
71 +
72 +
73 # page attention ops
74 def paged_attention_v1(
75 out: torch.Tensor,
vllm/model_executor/layers/activation.py
+16
@@ -141,6 +141,21 @@ class FastGELU(CustomOp):
141 return out
142
143
144 +class QuickGELU(CustomOp):
145 +
146 + # https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
147 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
148 + """PyTorch-native implementation equivalent to forward()."""
149 + return x * torch.sigmoid(1.702 * x)
150 +
151 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
152 + from vllm import _custom_ops as ops
153 +
154 + out = torch.empty_like(x)
155 + ops.gelu_quick(out, x)
156 + return out
157 +
158 +
159 class ScaledActivation(nn.Module):
160 """An activation function with post-scale parameters.
161
@@ -189,6 +204,7 @@ _ACTIVATION_REGISTRY = {
204 "gelu_new": NewGELU(),
205 "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
206 "relu": nn.ReLU(),
207 + "quick_gelu": QuickGELU(),
208 }
209
210
vllm/model_executor/models/clip.py new
+203
@@ -0,0 +1,203 @@
1 +"""Minimal implementation of CLIPVisionModel intended to be only used
2 +within a vision language model."""
3 +from typing import Optional, Tuple
4 +
5 +import torch
6 +import torch.nn as nn
7 +from transformers import CLIPVisionConfig
8 +from transformers.models.clip.modeling_clip import CLIPAttention
9 +
10 +from vllm.model_executor.layers.activation import get_act_fn
11 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
12 + RowParallelLinear)
13 +from vllm.model_executor.layers.quantization.base_config import (
14 + QuantizationConfig)
15 +
16 +
17 +def get_clip_num_patches(image_size: int, patch_size: int) -> int:
18 + assert image_size % patch_size == 0
19 + return (image_size // patch_size)**2
20 +
21 +
22 +# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
23 +class CLIPVisionEmbeddings(nn.Module):
24 +
25 + def __init__(self, config: CLIPVisionConfig):
26 + super().__init__()
27 + self.config = config
28 + self.embed_dim = config.hidden_size
29 + self.image_size = config.image_size
30 + self.patch_size = config.patch_size
31 +
32 + self.class_embedding = nn.Parameter(torch.randn(self.embed_dim))
33 +
34 + self.patch_embedding = nn.Conv2d(
35 + in_channels=config.num_channels,
36 + out_channels=self.embed_dim,
37 + kernel_size=self.patch_size,
38 + stride=self.patch_size,
39 + bias=False,
40 + )
41 +
42 + self.num_patches = get_clip_num_patches(self.image_size,
43 + self.patch_size)
44 + self.num_positions = self.num_patches + 1
45 + self.position_embedding = nn.Embedding(self.num_positions,
46 + self.embed_dim)
47 + self.register_buffer("position_ids",
48 + torch.arange(self.num_positions).expand((1, -1)),
49 + persistent=False)
50 +
51 + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
52 + batch_size = pixel_values.shape[0]
53 + target_dtype = self.patch_embedding.weight.dtype
54 + patch_embeds = self.patch_embedding(pixel_values.to(
55 + dtype=target_dtype)) # shape = [*, width, grid, grid]
56 + patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
57 +
58 + class_embeds = self.class_embedding.expand(batch_size, 1, -1)
59 + embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
60 + embeddings = embeddings + self.position_embedding(self.position_ids)
61 +
62 + return embeddings
63 +
64 +
65 +class CLIPMLP(nn.Module):
66 +
67 + def __init__(self,
68 + config: CLIPVisionConfig,
69 + quant_config: Optional[QuantizationConfig] = None):
70 + super().__init__()
71 + self.config = config
72 + self.activation_fn = get_act_fn(config.hidden_act)
73 + self.fc1 = ColumnParallelLinear(config.hidden_size,
74 + config.intermediate_size,
75 + bias=True,
76 + quant_config=quant_config)
77 + self.fc2 = RowParallelLinear(config.intermediate_size,
78 + config.hidden_size,
79 + bias=True,
80 + quant_config=quant_config)
81 +
82 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
83 + hidden_states, _ = self.fc1(hidden_states)
84 + hidden_states = self.activation_fn(hidden_states)
85 + hidden_states, _ = self.fc2(hidden_states)
86 +
87 + return hidden_states
88 +
89 +
90 +class CLIPEncoderLayer(nn.Module):
91 +
92 + def __init__(self,
93 + config: CLIPVisionConfig,
94 + quant_config: Optional[QuantizationConfig] = None):
95 + super().__init__()
96 +
97 + self.self_attn = CLIPAttention(config)
98 + self.layer_norm1 = nn.LayerNorm(config.hidden_size,
99 + eps=config.layer_norm_eps)
100 + self.mlp = CLIPMLP(config, quant_config=quant_config)
101 + self.layer_norm2 = nn.LayerNorm(config.hidden_size,
102 + eps=config.layer_norm_eps)
103 +
104 + def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor]:
105 +
106 + residual = hidden_states
107 +
108 + hidden_states = self.layer_norm1(hidden_states)
109 + hidden_states, _ = self.self_attn(hidden_states=hidden_states)
110 + hidden_states = residual + hidden_states
111 +
112 + residual = hidden_states
113 + hidden_states = self.layer_norm2(hidden_states)
114 + hidden_states = self.mlp(hidden_states)
115 + hidden_states = residual + hidden_states
116 +
117 + return hidden_states
118 +
119 +
120 +class CLIPEncoder(nn.Module):
121 + """
122 + Transformer encoder consisting of `config.num_hidden_layers` self
123 + attention layers. Each layer is a [`CLIPEncoderLayer`].
124 +
125 + Args:
126 + config: CLIPConfig
127 + """
128 +
129 + def __init__(self,
130 + config: CLIPVisionConfig,
131 + quant_config: Optional[QuantizationConfig] = None):
132 + super().__init__()
133 + self.config = config
134 + self.layers = nn.ModuleList([
135 + CLIPEncoderLayer(config=config, quant_config=quant_config)
136 + for _ in range(config.num_hidden_layers)
137 + ])
138 +
139 + def forward(self,
140 + inputs_embeds: torch.Tensor,
141 + vision_feature_layer: int = -1):
142 +
143 + # Encoder forward pass only up to the required layer
144 + num_layer = len(self.layers) + vision_feature_layer + 1
145 + hidden_states = inputs_embeds
146 + for encoder_layer in self.layers[:num_layer]:
147 + hidden_states = encoder_layer(hidden_states)
148 +
149 + return hidden_states
150 +
151 +
152 +class CLIPVisionTransformer(nn.Module):
153 +
154 + def __init__(self,
155 + config: CLIPVisionConfig,
156 + quant_config: Optional[QuantizationConfig] = None):
157 + super().__init__()
158 + self.config = config
159 + embed_dim = config.hidden_size
160 +
161 + self.embeddings = CLIPVisionEmbeddings(config)
162 +
163 + # NOTE: This typo of "layrnorm" is not fixed on purpose to match
164 + # the original transformers code and name of the model weights.
165 + self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
166 + self.encoder = CLIPEncoder(config=config, quant_config=quant_config)
167 +
168 + def forward(
169 + self,
170 + pixel_values: torch.Tensor,
171 + vision_feature_layer: int = -1,
172 + ) -> torch.Tensor:
173 +
174 + hidden_states = self.embeddings(pixel_values)
175 + hidden_states = self.pre_layrnorm(hidden_states)
176 + hidden_states = self.encoder(inputs_embeds=hidden_states,
177 + vision_feature_layer=vision_feature_layer)
178 +
179 + return hidden_states
180 +
181 +
182 +class CLIPVisionModel(nn.Module):
183 +
184 + config_class = CLIPVisionConfig
185 + main_input_name = "pixel_values"
186 +
187 + def __init__(self,
188 + config: CLIPVisionConfig,
189 + quant_config: Optional[QuantizationConfig] = None):
190 + super().__init__()
191 + self.vision_model = CLIPVisionTransformer(config=config,
192 + quant_config=quant_config)
193 +
194 + def forward(self,
195 + pixel_values: Optional[torch.Tensor] = None,
196 + vision_feature_layer: int = -1):
197 +
198 + return self.vision_model(pixel_values=pixel_values,
199 + vision_feature_layer=vision_feature_layer)
200 +
201 + @property
202 + def device(self):
203 + return next(self.parameters()).device
vllm/model_executor/models/llava.py
+9 -8
@@ -2,9 +2,7 @@ from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3 import torch
4 import torch.nn as nn
5 -# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
6 -# transformers' impl.
7 -from transformers import CLIPVisionModel, LlavaConfig
5 +from transformers import LlavaConfig
6
7 from vllm.attention import AttentionMetadata
8 from vllm.config import CacheConfig, VisionLanguageConfig
@@ -15,6 +13,7 @@ from vllm.model_executor.layers.quantization.base_config import (
13 from vllm.model_executor.layers.sampler import Sampler
14 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 +from vllm.model_executor.models.clip import CLIPVisionModel
17 from vllm.model_executor.models.llama import LlamaModel
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -189,12 +188,11 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
188
189 def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
190 pixel_values: torch.Tensor) -> torch.Tensor:
192 - # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
193 - image_outputs = vision_tower(pixel_values.to(vision_tower.device),
194 - output_hidden_states=True)
191
196 - image_features = image_outputs.hidden_states[
197 - self.config.vision_feature_layer]
192 + # NOTE: we skip the step to select the vision feature layer since
193 + # this is already done inside the vision tower
194 + image_features = vision_tower(pixel_values.to(vision_tower.device),
195 + self.config.vision_feature_layer)
196
197 return self._select_image_features(
198 image_features,
@@ -317,6 +315,9 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
315 for name, loaded_weight in weights:
316 if "rotary_emb.inv_freq" in name:
317 continue
318 + # post_layernorm is not needed in CLIPVisionModel
319 + if "vision_model.post_layernorm" in name:
320 + continue
321 for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
322 if key_to_modify in name:
323 name = name.replace(key_to_modify, new_key)
vllm/model_executor/models/llava_next.py
+10 -9
@@ -4,9 +4,7 @@ from typing import (Dict, Iterable, List, Literal, Optional, Tuple, TypedDict,
4 import torch
5 import torch.nn as nn
6 from PIL import Image
7 -# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
8 -# transformers' impl.
9 -from transformers import CLIPVisionModel, LlavaNextConfig
7 +from transformers import LlavaNextConfig
8 from transformers.models.llava_next.modeling_llava_next import (
9 get_anyres_image_grid_shape, unpad_image)
10 from typing_extensions import NotRequired
@@ -20,6 +18,7 @@ from vllm.model_executor.layers.quantization.base_config import (
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 +from vllm.model_executor.models.clip import CLIPVisionModel
22 from vllm.model_executor.models.llama import LlamaModel
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
@@ -121,7 +120,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
120
121 if self.vision_language_config.image_input_type == (
122 VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
124 - self.vision_tower = CLIPVisionModel(config.vision_config)
123 + self.vision_tower = CLIPVisionModel(config=config.vision_config)
124 else:
125 raise TypeError("Image features are not supported by LLaVA-NeXT")
126
@@ -219,12 +218,11 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
218
219 def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
220 pixel_values: torch.Tensor) -> torch.Tensor:
222 - # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
223 - image_outputs = vision_tower(pixel_values.to(vision_tower.device),
224 - output_hidden_states=True)
221
226 - image_features = image_outputs.hidden_states[
227 - self.config.vision_feature_layer]
222 + # NOTE: we skip the step to select the vision feature layer since
223 + # this is already done inside the vision tower
224 + image_features = vision_tower(pixel_values.to(vision_tower.device),
225 + self.config.vision_feature_layer)
226
227 return self._select_image_features(
228 image_features,
@@ -430,6 +428,9 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
428 for name, loaded_weight in weights:
429 if "rotary_emb.inv_freq" in name:
430 continue
431 + # post_layernorm is not needed in CLIPVisionModel
432 + if "vision_model.post_layernorm" in name:
433 + continue
434 for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
435 if key_to_modify in name:
436 name = name.replace(key_to_modify, new_key)
vllm/model_executor/models/phi3v.py
+9 -4
@@ -17,7 +17,7 @@ from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
17
18 import torch
19 import torch.nn as nn
20 -from transformers import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig
20 +from transformers import CLIPVisionConfig, PretrainedConfig
21
22 from vllm.attention import AttentionMetadata
23 from vllm.config import CacheConfig, VisionLanguageConfig
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import (
27 from vllm.model_executor.layers.sampler import Sampler
28 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 +from vllm.model_executor.models.clip import CLIPVisionModel
31 from vllm.model_executor.models.llama import LlamaModel
32 from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -70,9 +71,10 @@ class Phi3ImageEmbeddingBase(nn.Module):
71 LAYER_IDX = self.layer_idx
72 TYPE_FEATURE = self.type_feature
73
73 - img_processor_output = self.img_processor(img_embeds,
74 - output_hidden_states=True)
75 - img_feature = img_processor_output.hidden_states[LAYER_IDX]
74 + # NOTE: we skip the step to select the vision feature layer since
75 + # this is already done inside the img_processor
76 + img_feature = self.img_processor(img_embeds,
77 + vision_feature_layer=LAYER_IDX)
78
79 if TYPE_FEATURE == "patch":
80 patch_feature = img_feature[:, 1:]
@@ -352,6 +354,9 @@ class Phi3VForCausalLM(VisionLanguageModelBase):
354 for name, loaded_weight in weights:
355 if "rotary_emb.inv_freq" in name:
356 continue
357 + # post_layernorm is not needed in CLIPVisionModel
358 + if "vision_model.post_layernorm" in name:
359 + continue
360 for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
361 if key_to_modify in name:
362 name = name.replace(key_to_modify, new_key)