A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] Port over CLIPVisionModel for VLMs (#5591)
Roger Wang committed
Jun 20, 2024 at 04:52 UTC
ad137cd1112ab9b17ac36fc123fc7806a1d7473d
9 files changed
+269
-21
csrc/activation_kernels.cu
+12
@@ -135,6 +135,12 @@ __device__ __forceinline__ T gelu_fast_kernel(const T& x) {
135
return ((T)0.5) * x * (((T)1.0) + t);
136
}
137
138
+template <typename T>
139
+__device__ __forceinline__ T gelu_quick_kernel(const T& x) {
140
+ // x * sigmoid(1.702 * x)
141
+ return (T)(((float)x) / (1.0f + expf(-1.702f * (float)x)));
142
+}
143
+
144
} // namespace vllm
145
146
void gelu_new(torch::Tensor& out, // [..., d]
@@ -148,3 +154,9 @@ void gelu_fast(torch::Tensor& out, // [..., d]
154
{
155
LAUNCH_ACTIVATION_KERNEL(vllm::gelu_fast_kernel);
156
}
157
+
158
+void gelu_quick(torch::Tensor& out, // [..., d]
159
+ torch::Tensor& input) // [..., d]
160
+{
161
+ LAUNCH_ACTIVATION_KERNEL(vllm::gelu_quick_kernel);
162
+}
csrc/ops.h
+2
@@ -49,6 +49,8 @@ void gelu_new(torch::Tensor& out, torch::Tensor& input);
49
50
void gelu_fast(torch::Tensor& out, torch::Tensor& input);
51
52
+void gelu_quick(torch::Tensor& out, torch::Tensor& input);
53
+
54
#ifndef USE_ROCM
55
torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
56
const torch::Tensor& codebooks,
csrc/torch_bindings.cpp
+4
@@ -68,6 +68,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
68
ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
69
ops.impl("gelu_fast", torch::kCUDA, &gelu_fast);
70
71
+ // Quick GELU implementation.
72
+ ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
73
+ ops.impl("gelu_quick", torch::kCUDA, &gelu_quick);
74
+
75
// Layernorm
76
// Apply Root Mean Square (RMS) Normalization to the input tensor.
77
ops.def(
vllm/_custom_ops.py
+4
@@ -66,6 +66,10 @@ def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
66
torch.ops._C.gelu_new(out, x)
67
68
69
+def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
70
+ torch.ops._C.gelu_quick(out, x)
71
+
72
+
73
# page attention ops
74
def paged_attention_v1(
75
out: torch.Tensor,
vllm/model_executor/layers/activation.py
+16
@@ -141,6 +141,21 @@ class FastGELU(CustomOp):
141
return out
142
143
144
+class QuickGELU(CustomOp):
145
+
146
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
147
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
148
+ """PyTorch-native implementation equivalent to forward()."""
149
+ return x * torch.sigmoid(1.702 * x)
150
+
151
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
152
+ from vllm import _custom_ops as ops
153
+
154
+ out = torch.empty_like(x)
155
+ ops.gelu_quick(out, x)
156
+ return out
157
+
158
+
159
class ScaledActivation(nn.Module):
160
"""An activation function with post-scale parameters.
161
@@ -189,6 +204,7 @@ _ACTIVATION_REGISTRY = {
204
"gelu_new": NewGELU(),
205
"gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
206
"relu": nn.ReLU(),
207
+ "quick_gelu": QuickGELU(),
208
}
209
210
vllm/model_executor/models/clip.py
new
+203
@@ -0,0 +1,203 @@
1
+"""Minimal implementation of CLIPVisionModel intended to be only used
2
+within a vision language model."""
3
+from typing import Optional, Tuple
4
+
5
+import torch
6
+import torch.nn as nn
7
+from transformers import CLIPVisionConfig
8
+from transformers.models.clip.modeling_clip import CLIPAttention
9
+
10
+from vllm.model_executor.layers.activation import get_act_fn
11
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
12
+ RowParallelLinear)
13
+from vllm.model_executor.layers.quantization.base_config import (
14
+ QuantizationConfig)
15
+
16
+
17
+def get_clip_num_patches(image_size: int, patch_size: int) -> int:
18
+ assert image_size % patch_size == 0
19
+ return (image_size // patch_size)**2
20
+
21
+
22
+# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
23
+class CLIPVisionEmbeddings(nn.Module):
24
+
25
+ def __init__(self, config: CLIPVisionConfig):
26
+ super().__init__()
27
+ self.config = config
28
+ self.embed_dim = config.hidden_size
29
+ self.image_size = config.image_size
30
+ self.patch_size = config.patch_size
31
+
32
+ self.class_embedding = nn.Parameter(torch.randn(self.embed_dim))
33
+
34
+ self.patch_embedding = nn.Conv2d(
35
+ in_channels=config.num_channels,
36
+ out_channels=self.embed_dim,
37
+ kernel_size=self.patch_size,
38
+ stride=self.patch_size,
39
+ bias=False,
40
+ )
41
+
42
+ self.num_patches = get_clip_num_patches(self.image_size,
43
+ self.patch_size)
44
+ self.num_positions = self.num_patches + 1
45
+ self.position_embedding = nn.Embedding(self.num_positions,
46
+ self.embed_dim)
47
+ self.register_buffer("position_ids",
48
+ torch.arange(self.num_positions).expand((1, -1)),
49
+ persistent=False)
50
+
51
+ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
52
+ batch_size = pixel_values.shape[0]
53
+ target_dtype = self.patch_embedding.weight.dtype
54
+ patch_embeds = self.patch_embedding(pixel_values.to(
55
+ dtype=target_dtype)) # shape = [*, width, grid, grid]
56
+ patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
57
+
58
+ class_embeds = self.class_embedding.expand(batch_size, 1, -1)
59
+ embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
60
+ embeddings = embeddings + self.position_embedding(self.position_ids)
61
+
62
+ return embeddings
63
+
64
+
65
+class CLIPMLP(nn.Module):
66
+
67
+ def __init__(self,
68
+ config: CLIPVisionConfig,
69
+ quant_config: Optional[QuantizationConfig] = None):
70
+ super().__init__()
71
+ self.config = config
72
+ self.activation_fn = get_act_fn(config.hidden_act)
73
+ self.fc1 = ColumnParallelLinear(config.hidden_size,
74
+ config.intermediate_size,
75
+ bias=True,
76
+ quant_config=quant_config)
77
+ self.fc2 = RowParallelLinear(config.intermediate_size,
78
+ config.hidden_size,
79
+ bias=True,
80
+ quant_config=quant_config)
81
+
82
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
83
+ hidden_states, _ = self.fc1(hidden_states)
84
+ hidden_states = self.activation_fn(hidden_states)
85
+ hidden_states, _ = self.fc2(hidden_states)
86
+
87
+ return hidden_states
88
+
89
+
90
+class CLIPEncoderLayer(nn.Module):
91
+
92
+ def __init__(self,
93
+ config: CLIPVisionConfig,
94
+ quant_config: Optional[QuantizationConfig] = None):
95
+ super().__init__()
96
+
97
+ self.self_attn = CLIPAttention(config)
98
+ self.layer_norm1 = nn.LayerNorm(config.hidden_size,
99
+ eps=config.layer_norm_eps)
100
+ self.mlp = CLIPMLP(config, quant_config=quant_config)
101
+ self.layer_norm2 = nn.LayerNorm(config.hidden_size,
102
+ eps=config.layer_norm_eps)
103
+
104
+ def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor]:
105
+
106
+ residual = hidden_states
107
+
108
+ hidden_states = self.layer_norm1(hidden_states)
109
+ hidden_states, _ = self.self_attn(hidden_states=hidden_states)
110
+ hidden_states = residual + hidden_states
111
+
112
+ residual = hidden_states
113
+ hidden_states = self.layer_norm2(hidden_states)
114
+ hidden_states = self.mlp(hidden_states)
115
+ hidden_states = residual + hidden_states
116
+
117
+ return hidden_states
118
+
119
+
120
+class CLIPEncoder(nn.Module):
121
+ """
122
+ Transformer encoder consisting of `config.num_hidden_layers` self
123
+ attention layers. Each layer is a [`CLIPEncoderLayer`].
124
+
125
+ Args:
126
+ config: CLIPConfig
127
+ """
128
+
129
+ def __init__(self,
130
+ config: CLIPVisionConfig,
131
+ quant_config: Optional[QuantizationConfig] = None):
132
+ super().__init__()
133
+ self.config = config
134
+ self.layers = nn.ModuleList([
135
+ CLIPEncoderLayer(config=config, quant_config=quant_config)
136
+ for _ in range(config.num_hidden_layers)
137
+ ])
138
+
139
+ def forward(self,
140
+ inputs_embeds: torch.Tensor,
141
+ vision_feature_layer: int = -1):
142
+
143
+ # Encoder forward pass only up to the required layer
144
+ num_layer = len(self.layers) + vision_feature_layer + 1
145
+ hidden_states = inputs_embeds
146
+ for encoder_layer in self.layers[:num_layer]:
147
+ hidden_states = encoder_layer(hidden_states)
148
+
149
+ return hidden_states
150
+
151
+
152
+class CLIPVisionTransformer(nn.Module):
153
+
154
+ def __init__(self,
155
+ config: CLIPVisionConfig,
156
+ quant_config: Optional[QuantizationConfig] = None):
157
+ super().__init__()
158
+ self.config = config
159
+ embed_dim = config.hidden_size
160
+
161
+ self.embeddings = CLIPVisionEmbeddings(config)
162
+
163
+ # NOTE: This typo of "layrnorm" is not fixed on purpose to match
164
+ # the original transformers code and name of the model weights.
165
+ self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
166
+ self.encoder = CLIPEncoder(config=config, quant_config=quant_config)
167
+
168
+ def forward(
169
+ self,
170
+ pixel_values: torch.Tensor,
171
+ vision_feature_layer: int = -1,
172
+ ) -> torch.Tensor:
173
+
174
+ hidden_states = self.embeddings(pixel_values)
175
+ hidden_states = self.pre_layrnorm(hidden_states)
176
+ hidden_states = self.encoder(inputs_embeds=hidden_states,
177
+ vision_feature_layer=vision_feature_layer)
178
+
179
+ return hidden_states
180
+
181
+
182
+class CLIPVisionModel(nn.Module):
183
+
184
+ config_class = CLIPVisionConfig
185
+ main_input_name = "pixel_values"
186
+
187
+ def __init__(self,
188
+ config: CLIPVisionConfig,
189
+ quant_config: Optional[QuantizationConfig] = None):
190
+ super().__init__()
191
+ self.vision_model = CLIPVisionTransformer(config=config,
192
+ quant_config=quant_config)
193
+
194
+ def forward(self,
195
+ pixel_values: Optional[torch.Tensor] = None,
196
+ vision_feature_layer: int = -1):
197
+
198
+ return self.vision_model(pixel_values=pixel_values,
199
+ vision_feature_layer=vision_feature_layer)
200
+
201
+ @property
202
+ def device(self):
203
+ return next(self.parameters()).device
vllm/model_executor/models/llava.py
+9
-8
@@ -2,9 +2,7 @@ from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3
import torch
4
import torch.nn as nn
5
-# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
6
-# transformers' impl.
7
-from transformers import CLIPVisionModel, LlavaConfig
5
+from transformers import LlavaConfig
6
7
from vllm.attention import AttentionMetadata
8
from vllm.config import CacheConfig, VisionLanguageConfig
@@ -15,6 +13,7 @@ from vllm.model_executor.layers.quantization.base_config import (
13
from vllm.model_executor.layers.sampler import Sampler
14
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
+from vllm.model_executor.models.clip import CLIPVisionModel
17
from vllm.model_executor.models.llama import LlamaModel
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -189,12 +188,11 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
188
189
def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
190
pixel_values: torch.Tensor) -> torch.Tensor:
192
- # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
193
- image_outputs = vision_tower(pixel_values.to(vision_tower.device),
194
- output_hidden_states=True)
191
196
- image_features = image_outputs.hidden_states[
197
- self.config.vision_feature_layer]
192
+ # NOTE: we skip the step to select the vision feature layer since
193
+ # this is already done inside the vision tower
194
+ image_features = vision_tower(pixel_values.to(vision_tower.device),
195
+ self.config.vision_feature_layer)
196
197
return self._select_image_features(
198
image_features,
@@ -317,6 +315,9 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
315
for name, loaded_weight in weights:
316
if "rotary_emb.inv_freq" in name:
317
continue
318
+ # post_layernorm is not needed in CLIPVisionModel
319
+ if "vision_model.post_layernorm" in name:
320
+ continue
321
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
322
if key_to_modify in name:
323
name = name.replace(key_to_modify, new_key)
vllm/model_executor/models/llava_next.py
+10
-9
@@ -4,9 +4,7 @@ from typing import (Dict, Iterable, List, Literal, Optional, Tuple, TypedDict,
4
import torch
5
import torch.nn as nn
6
from PIL import Image
7
-# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
8
-# transformers' impl.
9
-from transformers import CLIPVisionModel, LlavaNextConfig
7
+from transformers import LlavaNextConfig
8
from transformers.models.llava_next.modeling_llava_next import (
9
get_anyres_image_grid_shape, unpad_image)
10
from typing_extensions import NotRequired
@@ -20,6 +18,7 @@ from vllm.model_executor.layers.quantization.base_config import (
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
+from vllm.model_executor.models.clip import CLIPVisionModel
22
from vllm.model_executor.models.llama import LlamaModel
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
@@ -121,7 +120,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
120
121
if self.vision_language_config.image_input_type == (
122
VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
124
- self.vision_tower = CLIPVisionModel(config.vision_config)
123
+ self.vision_tower = CLIPVisionModel(config=config.vision_config)
124
else:
125
raise TypeError("Image features are not supported by LLaVA-NeXT")
126
@@ -219,12 +218,11 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
218
219
def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
220
pixel_values: torch.Tensor) -> torch.Tensor:
222
- # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
223
- image_outputs = vision_tower(pixel_values.to(vision_tower.device),
224
- output_hidden_states=True)
221
226
- image_features = image_outputs.hidden_states[
227
- self.config.vision_feature_layer]
222
+ # NOTE: we skip the step to select the vision feature layer since
223
+ # this is already done inside the vision tower
224
+ image_features = vision_tower(pixel_values.to(vision_tower.device),
225
+ self.config.vision_feature_layer)
226
227
return self._select_image_features(
228
image_features,
@@ -430,6 +428,9 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
428
for name, loaded_weight in weights:
429
if "rotary_emb.inv_freq" in name:
430
continue
431
+ # post_layernorm is not needed in CLIPVisionModel
432
+ if "vision_model.post_layernorm" in name:
433
+ continue
434
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
435
if key_to_modify in name:
436
name = name.replace(key_to_modify, new_key)
vllm/model_executor/models/phi3v.py
+9
-4
@@ -17,7 +17,7 @@ from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
17
18
import torch
19
import torch.nn as nn
20
-from transformers import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig
20
+from transformers import CLIPVisionConfig, PretrainedConfig
21
22
from vllm.attention import AttentionMetadata
23
from vllm.config import CacheConfig, VisionLanguageConfig
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import (
27
from vllm.model_executor.layers.sampler import Sampler
28
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
+from vllm.model_executor.models.clip import CLIPVisionModel
31
from vllm.model_executor.models.llama import LlamaModel
32
from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -70,9 +71,10 @@ class Phi3ImageEmbeddingBase(nn.Module):
71
LAYER_IDX = self.layer_idx
72
TYPE_FEATURE = self.type_feature
73
73
- img_processor_output = self.img_processor(img_embeds,
74
- output_hidden_states=True)
75
- img_feature = img_processor_output.hidden_states[LAYER_IDX]
74
+ # NOTE: we skip the step to select the vision feature layer since
75
+ # this is already done inside the img_processor
76
+ img_feature = self.img_processor(img_embeds,
77
+ vision_feature_layer=LAYER_IDX)
78
79
if TYPE_FEATURE == "patch":
80
patch_feature = img_feature[:, 1:]
@@ -352,6 +354,9 @@ class Phi3VForCausalLM(VisionLanguageModelBase):
354
for name, loaded_weight in weights:
355
if "rotary_emb.inv_freq" in name:
356
continue
357
+ # post_layernorm is not needed in CLIPVisionModel
358
+ if "vision_model.post_layernorm" in name:
359
+ continue
360
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
361
if key_to_modify in name:
362
name = name.replace(key_to_modify, new_key)