A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix img_sizes Parsing in Phi3-Vision (#5888)
Roger Wang committed
Jun 27, 2024 at 01:29 UTC
2061f0b8a7f1a01683c4045096a092eedf6387a4
1 file changed
+6
-20
vllm/model_executor/models/phi3v.py
+6
-20
@@ -65,12 +65,6 @@ class Phi3ImageEmbeddingBase(nn.Module):
65
self.type_feature: str
66
self.img_processor: CLIPVisionModel
67
68
- def set_img_features(self, img_features: torch.FloatTensor) -> None:
69
- self.img_features = img_features
70
-
71
- def set_img_sizes(self, img_sizes: torch.LongTensor) -> None:
72
- self.img_sizes = img_sizes
73
-
68
def get_img_features(self,
69
img_embeds: torch.FloatTensor) -> torch.FloatTensor:
70
LAYER_IDX = self.layer_idx
@@ -144,21 +138,16 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
138
self.layer_idx = config.img_processor.get('layer_idx', -2)
139
self.type_feature = config.img_processor.get('type_feature', 'patch')
140
147
- def forward(self,
148
- input_ids: torch.LongTensor,
141
+ def forward(self, input_ids: torch.LongTensor,
142
pixel_values: torch.FloatTensor,
150
- image_sizes=None) -> torch.FloatTensor:
143
+ image_sizes: torch.Tensor) -> torch.FloatTensor:
144
"""process and merge text embeddings with image embeddings."""
145
146
+ # (batch_size, max_num_crops, 3, height, width)
147
img_embeds = pixel_values
154
- img_sizes = image_sizes
148
156
- if self.img_features is not None:
157
- img_embeds = self.img_features.clone()
158
- self.img_features = None
159
-
160
- if self.img_sizes is not None:
161
- img_sizes = self.img_sizes
149
+ # (batch_size, 2)
150
+ img_sizes = image_sizes
151
152
input_shape = input_ids.size()
153
input_ids = input_ids.view(-1, input_shape[-1])
@@ -190,11 +179,8 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
179
output_imgs = []
180
output_len = []
181
193
- if isinstance(img_sizes, torch.Tensor):
194
- img_sizes.squeeze_(0)
195
-
182
for _bs in range(bs):
197
- h, w = img_sizes
183
+ h, w = img_sizes[_bs]
184
h = h // 336
185
w = w // 336
186
B_ = h * w