A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Model] Refactor and decouple phi3v image embedding (#6621)
Isotr0py committed
Jul 22, 2024 at 07:07 UTC
25e778aa169ec871ad65524c793bf9bd43f6885a
1 file changed
+118
-119
vllm/model_executor/models/phi3v.py
+118
-119
@@ -43,6 +43,7 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
43
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
44
input_processor_for_clip)
45
from .interfaces import SupportsVision
46
+from .utils import merge_vision_embeddings
47
48
logger = init_logger(__name__)
49
@@ -71,9 +72,8 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
72
73
class Phi3ImageEmbeddingBase(nn.Module):
74
74
- def __init__(self, wte=None) -> None:
75
+ def __init__(self) -> None:
76
super().__init__()
76
- self.wte = wte
77
self.layer_idx: int
78
self.type_feature: str
79
self.img_processor: CLIPVisionModel
@@ -100,10 +100,9 @@ class Phi3ImageEmbeddingBase(nn.Module):
100
class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
101
"""Phi3 Image embedding with HD transform."""
102
103
- def __init__(self, config: PretrainedConfig, wte=None) -> None:
104
- super().__init__(wte)
103
+ def __init__(self, config: PretrainedConfig) -> None:
104
+ super().__init__()
105
106
- self.image_token_id = _IMAGE_TOKEN_ID
106
# n_embed or hidden_size
107
hidden_size = config.n_embd if hasattr(
108
config, 'n_embd') else config.hidden_size
@@ -149,118 +148,115 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
148
nn.Linear(dim_projection, dim_projection)])
149
self.img_projection = nn.Sequential(*layers)
150
152
- self.vocab_size = config.vocab_size
151
self.type_feature = config.img_processor.get('type_feature', 'patch')
152
155
- def forward(self, input_ids: torch.LongTensor,
156
- pixel_values: torch.FloatTensor,
153
+ def forward(self, pixel_values: torch.FloatTensor,
154
image_sizes: torch.Tensor) -> torch.FloatTensor:
158
- """process and merge text embeddings with image embeddings."""
159
-
160
- # (batch_size, max_num_crops, 3, height, width)
161
- img_embeds = pixel_values
162
-
163
- # (batch_size, 2)
164
- img_sizes = image_sizes
165
-
166
- input_shape = input_ids.size()
167
- input_ids = input_ids.view(-1, input_shape[-1])
168
-
169
- positions = torch.nonzero(input_ids == self.image_token_id)
170
-
171
- select = False
172
-
173
- target_dtype = self.img_projection[0].bias.dtype
174
-
175
- if len(positions.tolist()) > 0:
176
- # if self.use_hd_transform and img_sizes:
177
- # img_embeds: (num_images, max_num_crops, 3, H, W)
178
- # img_sizes: (num_images, 2).view(1, -1)
179
-
180
- bs = img_embeds.shape[0]
181
- # Nx(HW)xC
182
- img_features = self.get_img_features(img_embeds.flatten(0, 1))
183
- base_feat_height = base_feat_width = int(
184
- img_features.shape[1]**0.5)
185
-
186
- # bs x max_num_crops x (24x24) x C
187
- img_features = img_features.view(
188
- bs, -1, base_feat_height * base_feat_width, self.image_dim_out)
189
- C = self.image_dim_out
190
- H = base_feat_height
191
-
192
- output_imgs = []
193
- output_len = []
194
-
195
- for _bs in range(bs):
196
- h, w = img_sizes[_bs]
197
- h = h // 336
198
- w = w // 336
199
- B_ = h * w
200
-
201
- # 1 x (24x24) x 1024
202
- global_img_feature = img_features[_bs, :1]
203
-
204
- # 1 x 12 x 12 x 4096
205
- glb_img = global_img_feature \
206
- .reshape(1, H // 2, 2, H // 2, 2,C) \
207
- .permute(0, 1, 3, 2, 4, 5) \
208
- .reshape(1, H // 2, H // 2, 4 * C)
209
- temp_glb_GN = self.sub_GN.repeat(1, H // 2, 1, 1)
210
-
211
- # 1 x 156 x 4096
212
- glb_img = torch.cat([glb_img, temp_glb_GN],
213
- dim=2).reshape(1, -1, 4 * C)
214
-
215
- # (max_num_crops-1) x (12x12) x C
216
- sub_img = img_features[_bs, 1:]
217
- # 16x574x1024
218
- # get rid of padding sub_img
219
- sub_img = sub_img[:B_]
220
-
221
- sub_img = sub_img.reshape(B_, H // 2, 2, H // 2, 2, C) \
222
- .permute(0, 1, 3, 2, 4, 5).reshape(B_, -1, 4 * C)
223
- sub_img = sub_img.reshape(1, h, w, 12, 12, -1) \
224
- .permute(0, 1, 3, 2, 4, 5) \
225
- .reshape(1, h * 12, w * 12, 4 * C)
226
- temp_sub_GN = self.sub_GN.repeat(1, h * 12, 1, 1)
227
- sub_img = torch.cat([sub_img, temp_sub_GN],
228
- dim=2).reshape(1, -1, 4 * C)
229
- # (1, num_img_tokens, 1024*4)
230
-
231
- # glb + sub
232
- if self.hd_transform_order == 'glb_sub':
233
- output_imgs.append(
234
- torch.cat([glb_img, self.glb_GN, sub_img], dim=1))
235
- elif self.hd_transform_order == 'sub_glb':
236
- output_imgs.append(
237
- torch.cat([sub_img, self.glb_GN, glb_img], dim=1))
238
-
239
- temp_len = int((h * w + 1) * 144 + 1 + (h + 1) * 12)
240
- output_len.append(temp_len)
241
-
242
- num_img_tokens = output_len
243
- img_set_tensor = []
244
- for _output_img in output_imgs:
245
- img_feature_proj = self.img_projection(
246
- _output_img.to(target_dtype))
247
- img_set_tensor.append(img_feature_proj)
248
- select = True
249
-
250
- input_ids.clamp_min_(0).clamp_max_(self.vocab_size)
251
-
252
- hidden_states = self.wte(input_ids)
253
-
254
- if select:
255
- idx = 0
256
- for i, cnt in enumerate(num_img_tokens):
257
- hidden_states[positions[idx, 0],
258
- positions[idx, 1]:positions[idx, 1] +
259
- cnt] = (img_set_tensor[i].to(
260
- hidden_states.dtype))
261
- idx += cnt
262
-
263
- return hidden_states.squeeze(0)
155
+ """
156
+ process image and return vision embeddings.
157
+
158
+ pixel_values: (num_images, num_crops, c, h, w)
159
+ output: (num_images, num_img_tokens, hidden_size)
160
+ """
161
+ num_images, num_crops, c, h, w = pixel_values.shape
162
+ pixel_values = pixel_values.flatten(0, 1)
163
+ img_features = self.get_img_features(pixel_values)
164
+ img_features = img_features.reshape(num_images, num_crops, -1,
165
+ self.image_dim_out)
166
+ image_features_proj = self.hd_feature_transform(
167
+ img_features, image_sizes)
168
+ return image_features_proj
169
+
170
+ def hd_feature_transform(self, image_features, image_sizes):
171
+ """
172
+ image_features: (num_images, num_crops+1, 24*24, 1024)
173
+ """
174
+ assert (
175
+ self.hd_transform_order == 'sub_glb'
176
+ ), f'hd_transform_order `{self.hd_transform_order}` not implemented'
177
+ if isinstance(self.img_projection, nn.Sequential):
178
+ target_device = self.img_projection[0].bias.device
179
+ target_dtype = self.img_projection[0].bias.dtype
180
+ else: # It's a single nn.Linear layer
181
+ target_device = self.img_projection.bias.device
182
+ target_dtype = self.img_projection.bias.dtype
183
+
184
+ global_image_features = image_features[:,
185
+ 0] # (num_images, 24*24, 1024)
186
+ # global feature can be viewed as a special HD case with num_crops 1x1
187
+ global_image_features_hd = self.reshape_hd_patches_2x2merge(
188
+ global_image_features, 1, 1)
189
+ global_image_features_hd_newline = self.add_image_newline(
190
+ global_image_features_hd)
191
+
192
+ all_image_embeddings = []
193
+ # need a for loop to process each image because of different image sizes
194
+ # (patch arrangement is different for each image)
195
+ for i, img_size in enumerate(image_sizes):
196
+ h, w = img_size
197
+ h_crop = h // 336
198
+ w_crop = w // 336
199
+ num_crops = h_crop * w_crop
200
+
201
+ # NOTE: real num_crops is padded
202
+ # (num_crops, 24*24, 1024)
203
+ sub_image_features = image_features[i, 1:1 + num_crops]
204
+ sub_image_features_hd = self.reshape_hd_patches_2x2merge(
205
+ sub_image_features, h_crop, w_crop)
206
+ sub_image_features_hd_newline = self.add_image_newline(
207
+ sub_image_features_hd)
208
+
209
+ # [sub features, separator, global features]
210
+ all_image_embeddings.append(
211
+ torch.cat([
212
+ sub_image_features_hd_newline.squeeze(
213
+ 0), # (h_crop*12*(w_crop*12+1), 4096)
214
+ self.glb_GN.squeeze(0),
215
+ global_image_features_hd_newline[i],
216
+ ]))
217
+
218
+ image_features_proj = self.img_projection(
219
+ torch.stack(all_image_embeddings).to(target_device, target_dtype)
220
+ ) # (num_images, (h_crop*12*(w_crop*12+1)+1), hidden_size)
221
+
222
+ return image_features_proj
223
+
224
+ def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop):
225
+ """
226
+ image_features: (num_images*num_crops, 24*24, 1024)
227
+ output: (num_images, h_crop*12, w_crop*12, 4096)
228
+ where h_crop*w_crop == num_crops
229
+ """
230
+ N, L, C = image_features.shape
231
+ assert L == 576 and C == 1024 and N % (h_crop * w_crop) == 0
232
+ num_images = N // (h_crop * w_crop)
233
+ H = int(L**0.5)
234
+ image_features_hd = (
235
+ image_features.reshape(N, H, H, C) # N, 24, 24, 1024
236
+ .reshape(N, H // 2, 2, H // 2, 2, C) # N, 12, 2, 12, 2, 1024
237
+ .permute(0, 1, 3, 2, 4, 5) # N, 12, 12, 2, 2, 1024
238
+ .reshape(N, -1, 4 * C) # N, 144, 4096
239
+ .reshape(num_images, h_crop, w_crop, H // 2, H // 2,
240
+ -1) # n_img, h_crop, w_crop, 12, 12, 4096
241
+ .permute(0, 1, 3, 2, 4, 5) # n_img, h_crop, 12, w_crop, 12, 4096
242
+ .reshape(num_images, h_crop * H // 2, w_crop * H // 2,
243
+ 4 * C) # n_img, h_crop*12, w_crop*12, 4096
244
+ )
245
+ return image_features_hd
246
+
247
+ def add_image_newline(self, image_features_hd):
248
+ """
249
+ image_features_hd: (num_images, h_crop*12, w_crop*12, 4096)
250
+ output: (num_images, (h_crop*12) * (w_crop*12+1), 4096)
251
+ """
252
+ num_images, h, w, hid_dim = image_features_hd.shape
253
+ # add the newline token to the HD image feature patches
254
+ newline_embeddings = self.sub_GN.expand(num_images, h, -1,
255
+ -1) # (n_img, h, 1, hid_dim)
256
+ image_features_hd_newline = torch.cat(
257
+ [image_features_hd, newline_embeddings],
258
+ dim=2).reshape(num_images, -1, hid_dim)
259
+ return image_features_hd_newline
260
261
262
class Phi3VImagePixelInputs(TypedDict):
@@ -458,12 +454,12 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
454
455
self.config = config
456
self.multimodal_config = multimodal_config
457
+ self.image_token_id = _IMAGE_TOKEN_ID
458
459
self.model = LlamaModel(config, cache_config, quant_config)
460
461
# TODO: Optionally initializes this for supporting embeddings.
465
- self.vision_embed_tokens = Phi3HDImageEmbedding(
466
- config, self.model.embed_tokens)
462
+ self.vision_embed_tokens = Phi3HDImageEmbedding(config)
463
self.lm_head = ParallelLMHead(config.vocab_size,
464
config.hidden_size,
465
quant_config=quant_config)
@@ -530,9 +526,12 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
526
image_input = self._parse_and_validate_image_input(**kwargs)
527
528
if image_input is not None:
533
- inputs_embeds = self.vision_embed_tokens(
534
- input_ids, image_input["data"], image_input["image_sizes"])
535
-
529
+ vision_embeddings = self.vision_embed_tokens(
530
+ image_input["data"], image_input["image_sizes"])
531
+ inputs_embeds = self.model.get_input_embeddings(input_ids)
532
+ inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
533
+ vision_embeddings,
534
+ self.image_token_id)
535
input_ids = None
536
else:
537
inputs_embeds = None