@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4c5d8e8ea9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix phi3v batch inference when images have different aspect ratio (#7392)

Isotr0py committed Aug 11, 2024 at 00:19 UTC 4c5d8e8ea91aa19415aa479d81e818913d51414c
4 files changed +25 -19
tests/models/test_phi3v.py
+4 -1
@@ -81,7 +81,10 @@ def run_test(
81
82 inputs_per_image = [(
83 [prompt for _ in size_factors],
84 - [rescale_image_size(image, factor) for factor in size_factors],
84 + [
85 + rescale_image_size(image, factor, transpose=idx)
86 + for idx, factor in enumerate(size_factors)
87 + ],
88 ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
89
90 # NOTE: take care of the order. run vLLM first, and then run HF.
tests/tracing/test_tracing.py
+2 -2
@@ -114,5 +114,5 @@ def test_traces(trace_service):
114 SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115 e2e_time = metrics.finished_time - metrics.arrival_time
116 assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
117 - assert attributes.get(SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER
118 - ) == metrics.scheduler_time
117 + assert attributes.get(
118 + SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
vllm/model_executor/models/phi3v.py
+12 -14
@@ -189,7 +189,7 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
189 global_image_features_hd_newline = self.add_image_newline(
190 global_image_features_hd)
191
192 - all_image_embeddings = []
192 + batch_image_features_proj = []
193 # need a for loop to process each image because of different image sizes
194 # (patch arrangement is different for each image)
195 for i, img_size in enumerate(image_sizes):
@@ -207,19 +207,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
207 sub_image_features_hd)
208
209 # [sub features, separator, global features]
210 - all_image_embeddings.append(
211 - torch.cat([
212 - sub_image_features_hd_newline.squeeze(
213 - 0), # (h_crop*12*(w_crop*12+1), 4096)
214 - self.glb_GN.squeeze(0),
215 - global_image_features_hd_newline[i],
216 - ]))
217 -
218 - image_features_proj = self.img_projection(
219 - torch.stack(all_image_embeddings).to(target_device, target_dtype)
220 - ) # (num_images, (h_crop*12*(w_crop*12+1)+1), hidden_size)
221 -
222 - return image_features_proj
210 + image_embeddings = torch.cat([
211 + sub_image_features_hd_newline.squeeze(
212 + 0), # (h_crop*12*(w_crop*12+1), 4096)
213 + self.glb_GN.squeeze(0),
214 + global_image_features_hd_newline[i],
215 + ])
216 + img_proj = self.img_projection(
217 + image_embeddings.to(target_device, target_dtype))
218 + batch_image_features_proj.append(img_proj)
219 +
220 + return batch_image_features_proj
221
222 def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop):
223 """
vllm/multimodal/utils.py
+7 -2
@@ -90,8 +90,13 @@ def load_image_from_base64(image: Union[bytes, str]) -> Image.Image:
90 return _load_image_from_bytes(base64.b64decode(image))
91
92
93 -def rescale_image_size(image: Image.Image, size_factor: float) -> Image.Image:
93 +def rescale_image_size(image: Image.Image,
94 + size_factor: float,
95 + transpose: int = -1) -> Image.Image:
96 """Rescale the dimensions of an image by a constant factor."""
97 new_width = int(image.width * size_factor)
98 new_height = int(image.height * size_factor)
97 - return image.resize((new_width, new_height))
99 + image = image.resize((new_width, new_height))
100 + if transpose >= 0:
101 + image = image.transpose(Image.Transpose(transpose))
102 + return image