A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix phi3v batch inference when images have different aspect ratio (#7392)
Isotr0py committed
Aug 11, 2024 at 00:19 UTC
4c5d8e8ea91aa19415aa479d81e818913d51414c
4 files changed
+25
-19
tests/models/test_phi3v.py
+4
-1
@@ -81,7 +81,10 @@ def run_test(
81
82
inputs_per_image = [(
83
[prompt for _ in size_factors],
84
- [rescale_image_size(image, factor) for factor in size_factors],
84
+ [
85
+ rescale_image_size(image, factor, transpose=idx)
86
+ for idx, factor in enumerate(size_factors)
87
+ ],
88
) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
89
90
# NOTE: take care of the order. run vLLM first, and then run HF.
tests/tracing/test_tracing.py
+2
-2
@@ -114,5 +114,5 @@ def test_traces(trace_service):
114
SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115
e2e_time = metrics.finished_time - metrics.arrival_time
116
assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
117
- assert attributes.get(SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER
118
- ) == metrics.scheduler_time
117
+ assert attributes.get(
118
+ SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
vllm/model_executor/models/phi3v.py
+12
-14
@@ -189,7 +189,7 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
189
global_image_features_hd_newline = self.add_image_newline(
190
global_image_features_hd)
191
192
- all_image_embeddings = []
192
+ batch_image_features_proj = []
193
# need a for loop to process each image because of different image sizes
194
# (patch arrangement is different for each image)
195
for i, img_size in enumerate(image_sizes):
@@ -207,19 +207,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
207
sub_image_features_hd)
208
209
# [sub features, separator, global features]
210
- all_image_embeddings.append(
211
- torch.cat([
212
- sub_image_features_hd_newline.squeeze(
213
- 0), # (h_crop*12*(w_crop*12+1), 4096)
214
- self.glb_GN.squeeze(0),
215
- global_image_features_hd_newline[i],
216
- ]))
217
-
218
- image_features_proj = self.img_projection(
219
- torch.stack(all_image_embeddings).to(target_device, target_dtype)
220
- ) # (num_images, (h_crop*12*(w_crop*12+1)+1), hidden_size)
221
-
222
- return image_features_proj
210
+ image_embeddings = torch.cat([
211
+ sub_image_features_hd_newline.squeeze(
212
+ 0), # (h_crop*12*(w_crop*12+1), 4096)
213
+ self.glb_GN.squeeze(0),
214
+ global_image_features_hd_newline[i],
215
+ ])
216
+ img_proj = self.img_projection(
217
+ image_embeddings.to(target_device, target_dtype))
218
+ batch_image_features_proj.append(img_proj)
219
+
220
+ return batch_image_features_proj
221
222
def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop):
223
"""
vllm/multimodal/utils.py
+7
-2
@@ -90,8 +90,13 @@ def load_image_from_base64(image: Union[bytes, str]) -> Image.Image:
90
return _load_image_from_bytes(base64.b64decode(image))
91
92
93
-def rescale_image_size(image: Image.Image, size_factor: float) -> Image.Image:
93
+def rescale_image_size(image: Image.Image,
94
+ size_factor: float,
95
+ transpose: int = -1) -> Image.Image:
96
"""Rescale the dimensions of an image by a constant factor."""
97
new_width = int(image.width * size_factor)
98
new_height = int(image.height * size_factor)
97
- return image.resize((new_width, new_height))
99
+ image = image.resize((new_width, new_height))
100
+ if transpose >= 0:
101
+ image = image.transpose(Image.Transpose(transpose))
102
+ return image