@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 25e778aa16
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] Refactor and decouple phi3v image embedding (#6621)

Isotr0py committed Jul 22, 2024 at 07:07 UTC 25e778aa169ec871ad65524c793bf9bd43f6885a
1 file changed +118 -119
vllm/model_executor/models/phi3v.py
+118 -119
@@ -43,6 +43,7 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
43 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
44 input_processor_for_clip)
45 from .interfaces import SupportsVision
46 +from .utils import merge_vision_embeddings
47
48 logger = init_logger(__name__)
49
@@ -71,9 +72,8 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
72
73 class Phi3ImageEmbeddingBase(nn.Module):
74
74 - def __init__(self, wte=None) -> None:
75 + def __init__(self) -> None:
76 super().__init__()
76 - self.wte = wte
77 self.layer_idx: int
78 self.type_feature: str
79 self.img_processor: CLIPVisionModel
@@ -100,10 +100,9 @@ class Phi3ImageEmbeddingBase(nn.Module):
100 class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
101 """Phi3 Image embedding with HD transform."""
102
103 - def __init__(self, config: PretrainedConfig, wte=None) -> None:
104 - super().__init__(wte)
103 + def __init__(self, config: PretrainedConfig) -> None:
104 + super().__init__()
105
106 - self.image_token_id = _IMAGE_TOKEN_ID
106 # n_embed or hidden_size
107 hidden_size = config.n_embd if hasattr(
108 config, 'n_embd') else config.hidden_size
@@ -149,118 +148,115 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
148 nn.Linear(dim_projection, dim_projection)])
149 self.img_projection = nn.Sequential(*layers)
150
152 - self.vocab_size = config.vocab_size
151 self.type_feature = config.img_processor.get('type_feature', 'patch')
152
155 - def forward(self, input_ids: torch.LongTensor,
156 - pixel_values: torch.FloatTensor,
153 + def forward(self, pixel_values: torch.FloatTensor,
154 image_sizes: torch.Tensor) -> torch.FloatTensor:
158 - """process and merge text embeddings with image embeddings."""
159 -
160 - # (batch_size, max_num_crops, 3, height, width)
161 - img_embeds = pixel_values
162 -
163 - # (batch_size, 2)
164 - img_sizes = image_sizes
165 -
166 - input_shape = input_ids.size()
167 - input_ids = input_ids.view(-1, input_shape[-1])
168 -
169 - positions = torch.nonzero(input_ids == self.image_token_id)
170 -
171 - select = False
172 -
173 - target_dtype = self.img_projection[0].bias.dtype
174 -
175 - if len(positions.tolist()) > 0:
176 - # if self.use_hd_transform and img_sizes:
177 - # img_embeds: (num_images, max_num_crops, 3, H, W)
178 - # img_sizes: (num_images, 2).view(1, -1)
179 -
180 - bs = img_embeds.shape[0]
181 - # Nx(HW)xC
182 - img_features = self.get_img_features(img_embeds.flatten(0, 1))
183 - base_feat_height = base_feat_width = int(
184 - img_features.shape[1]**0.5)
185 -
186 - # bs x max_num_crops x (24x24) x C
187 - img_features = img_features.view(
188 - bs, -1, base_feat_height * base_feat_width, self.image_dim_out)
189 - C = self.image_dim_out
190 - H = base_feat_height
191 -
192 - output_imgs = []
193 - output_len = []
194 -
195 - for _bs in range(bs):
196 - h, w = img_sizes[_bs]
197 - h = h // 336
198 - w = w // 336
199 - B_ = h * w
200 -
201 - # 1 x (24x24) x 1024
202 - global_img_feature = img_features[_bs, :1]
203 -
204 - # 1 x 12 x 12 x 4096
205 - glb_img = global_img_feature \
206 - .reshape(1, H // 2, 2, H // 2, 2,C) \
207 - .permute(0, 1, 3, 2, 4, 5) \
208 - .reshape(1, H // 2, H // 2, 4 * C)
209 - temp_glb_GN = self.sub_GN.repeat(1, H // 2, 1, 1)
210 -
211 - # 1 x 156 x 4096
212 - glb_img = torch.cat([glb_img, temp_glb_GN],
213 - dim=2).reshape(1, -1, 4 * C)
214 -
215 - # (max_num_crops-1) x (12x12) x C
216 - sub_img = img_features[_bs, 1:]
217 - # 16x574x1024
218 - # get rid of padding sub_img
219 - sub_img = sub_img[:B_]
220 -
221 - sub_img = sub_img.reshape(B_, H // 2, 2, H // 2, 2, C) \
222 - .permute(0, 1, 3, 2, 4, 5).reshape(B_, -1, 4 * C)
223 - sub_img = sub_img.reshape(1, h, w, 12, 12, -1) \
224 - .permute(0, 1, 3, 2, 4, 5) \
225 - .reshape(1, h * 12, w * 12, 4 * C)
226 - temp_sub_GN = self.sub_GN.repeat(1, h * 12, 1, 1)
227 - sub_img = torch.cat([sub_img, temp_sub_GN],
228 - dim=2).reshape(1, -1, 4 * C)
229 - # (1, num_img_tokens, 1024*4)
230 -
231 - # glb + sub
232 - if self.hd_transform_order == 'glb_sub':
233 - output_imgs.append(
234 - torch.cat([glb_img, self.glb_GN, sub_img], dim=1))
235 - elif self.hd_transform_order == 'sub_glb':
236 - output_imgs.append(
237 - torch.cat([sub_img, self.glb_GN, glb_img], dim=1))
238 -
239 - temp_len = int((h * w + 1) * 144 + 1 + (h + 1) * 12)
240 - output_len.append(temp_len)
241 -
242 - num_img_tokens = output_len
243 - img_set_tensor = []
244 - for _output_img in output_imgs:
245 - img_feature_proj = self.img_projection(
246 - _output_img.to(target_dtype))
247 - img_set_tensor.append(img_feature_proj)
248 - select = True
249 -
250 - input_ids.clamp_min_(0).clamp_max_(self.vocab_size)
251 -
252 - hidden_states = self.wte(input_ids)
253 -
254 - if select:
255 - idx = 0
256 - for i, cnt in enumerate(num_img_tokens):
257 - hidden_states[positions[idx, 0],
258 - positions[idx, 1]:positions[idx, 1] +
259 - cnt] = (img_set_tensor[i].to(
260 - hidden_states.dtype))
261 - idx += cnt
262 -
263 - return hidden_states.squeeze(0)
155 + """
156 + process image and return vision embeddings.
157 +
158 + pixel_values: (num_images, num_crops, c, h, w)
159 + output: (num_images, num_img_tokens, hidden_size)
160 + """
161 + num_images, num_crops, c, h, w = pixel_values.shape
162 + pixel_values = pixel_values.flatten(0, 1)
163 + img_features = self.get_img_features(pixel_values)
164 + img_features = img_features.reshape(num_images, num_crops, -1,
165 + self.image_dim_out)
166 + image_features_proj = self.hd_feature_transform(
167 + img_features, image_sizes)
168 + return image_features_proj
169 +
170 + def hd_feature_transform(self, image_features, image_sizes):
171 + """
172 + image_features: (num_images, num_crops+1, 24*24, 1024)
173 + """
174 + assert (
175 + self.hd_transform_order == 'sub_glb'
176 + ), f'hd_transform_order `{self.hd_transform_order}` not implemented'
177 + if isinstance(self.img_projection, nn.Sequential):
178 + target_device = self.img_projection[0].bias.device
179 + target_dtype = self.img_projection[0].bias.dtype
180 + else: # It's a single nn.Linear layer
181 + target_device = self.img_projection.bias.device
182 + target_dtype = self.img_projection.bias.dtype
183 +
184 + global_image_features = image_features[:,
185 + 0] # (num_images, 24*24, 1024)
186 + # global feature can be viewed as a special HD case with num_crops 1x1
187 + global_image_features_hd = self.reshape_hd_patches_2x2merge(
188 + global_image_features, 1, 1)
189 + global_image_features_hd_newline = self.add_image_newline(
190 + global_image_features_hd)
191 +
192 + all_image_embeddings = []
193 + # need a for loop to process each image because of different image sizes
194 + # (patch arrangement is different for each image)
195 + for i, img_size in enumerate(image_sizes):
196 + h, w = img_size
197 + h_crop = h // 336
198 + w_crop = w // 336
199 + num_crops = h_crop * w_crop
200 +
201 + # NOTE: real num_crops is padded
202 + # (num_crops, 24*24, 1024)
203 + sub_image_features = image_features[i, 1:1 + num_crops]
204 + sub_image_features_hd = self.reshape_hd_patches_2x2merge(
205 + sub_image_features, h_crop, w_crop)
206 + sub_image_features_hd_newline = self.add_image_newline(
207 + sub_image_features_hd)
208 +
209 + # [sub features, separator, global features]
210 + all_image_embeddings.append(
211 + torch.cat([
212 + sub_image_features_hd_newline.squeeze(
213 + 0), # (h_crop*12*(w_crop*12+1), 4096)
214 + self.glb_GN.squeeze(0),
215 + global_image_features_hd_newline[i],
216 + ]))
217 +
218 + image_features_proj = self.img_projection(
219 + torch.stack(all_image_embeddings).to(target_device, target_dtype)
220 + ) # (num_images, (h_crop*12*(w_crop*12+1)+1), hidden_size)
221 +
222 + return image_features_proj
223 +
224 + def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop):
225 + """
226 + image_features: (num_images*num_crops, 24*24, 1024)
227 + output: (num_images, h_crop*12, w_crop*12, 4096)
228 + where h_crop*w_crop == num_crops
229 + """
230 + N, L, C = image_features.shape
231 + assert L == 576 and C == 1024 and N % (h_crop * w_crop) == 0
232 + num_images = N // (h_crop * w_crop)
233 + H = int(L**0.5)
234 + image_features_hd = (
235 + image_features.reshape(N, H, H, C) # N, 24, 24, 1024
236 + .reshape(N, H // 2, 2, H // 2, 2, C) # N, 12, 2, 12, 2, 1024
237 + .permute(0, 1, 3, 2, 4, 5) # N, 12, 12, 2, 2, 1024
238 + .reshape(N, -1, 4 * C) # N, 144, 4096
239 + .reshape(num_images, h_crop, w_crop, H // 2, H // 2,
240 + -1) # n_img, h_crop, w_crop, 12, 12, 4096
241 + .permute(0, 1, 3, 2, 4, 5) # n_img, h_crop, 12, w_crop, 12, 4096
242 + .reshape(num_images, h_crop * H // 2, w_crop * H // 2,
243 + 4 * C) # n_img, h_crop*12, w_crop*12, 4096
244 + )
245 + return image_features_hd
246 +
247 + def add_image_newline(self, image_features_hd):
248 + """
249 + image_features_hd: (num_images, h_crop*12, w_crop*12, 4096)
250 + output: (num_images, (h_crop*12) * (w_crop*12+1), 4096)
251 + """
252 + num_images, h, w, hid_dim = image_features_hd.shape
253 + # add the newline token to the HD image feature patches
254 + newline_embeddings = self.sub_GN.expand(num_images, h, -1,
255 + -1) # (n_img, h, 1, hid_dim)
256 + image_features_hd_newline = torch.cat(
257 + [image_features_hd, newline_embeddings],
258 + dim=2).reshape(num_images, -1, hid_dim)
259 + return image_features_hd_newline
260
261
262 class Phi3VImagePixelInputs(TypedDict):
@@ -458,12 +454,12 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
454
455 self.config = config
456 self.multimodal_config = multimodal_config
457 + self.image_token_id = _IMAGE_TOKEN_ID
458
459 self.model = LlamaModel(config, cache_config, quant_config)
460
461 # TODO: Optionally initializes this for supporting embeddings.
465 - self.vision_embed_tokens = Phi3HDImageEmbedding(
466 - config, self.model.embed_tokens)
462 + self.vision_embed_tokens = Phi3HDImageEmbedding(config)
463 self.lm_head = ParallelLMHead(config.vocab_size,
464 config.hidden_size,
465 quant_config=quant_config)
@@ -530,9 +526,12 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
526 image_input = self._parse_and_validate_image_input(**kwargs)
527
528 if image_input is not None:
533 - inputs_embeds = self.vision_embed_tokens(
534 - input_ids, image_input["data"], image_input["image_sizes"])
535 -
529 + vision_embeddings = self.vision_embed_tokens(
530 + image_input["data"], image_input["image_sizes"])
531 + inputs_embeds = self.model.get_input_embeddings(input_ids)
532 + inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
533 + vision_embeddings,
534 + self.image_token_id)
535 input_ids = None
536 else:
537 inputs_embeds = None