@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / edd5fe5fa2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Add phi3v resize for dynamic shape and fix torchvision requirement (#5772)

Isotr0py committed Jun 24, 2024 at 12:11 UTC edd5fe5fa29b8f9cc5fa37a30cc7211e0ff37067
5 files changed +72 -5
requirements-cpu.txt
+1
@@ -3,4 +3,5 @@
3
4 # Dependencies for x86_64 CPUs
5 torch == 2.3.1+cpu
6 +torchvision == 0.18.1+cpu # required for the image processor of phi3v, this must be updated alongside torch
7 triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
\ No newline at end of file
requirements-cuda.txt
+2
@@ -5,5 +5,7 @@
5 ray >= 2.9
6 nvidia-ml-py # for pynvml package
7 torch == 2.3.0
8 +# These must be updated alongside torch
9 +torchvision == 0.18.0 # Required for phi3v processor, also see https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
11 vllm-flash-attn == 2.5.9 # Requires PyTorch 2.3.0
requirements-test.txt
-1
@@ -14,7 +14,6 @@ peft
14 requests
15 ray
16 sentence-transformers # required for embedding
17 -torchvision # required for the image processor of phi3v
17
18 # Benchmarking
19 aiohttp
tests/models/test_phi3v.py
+4
@@ -22,6 +22,7 @@ assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
22 def iter_phi3v_configs(model_name: str):
23 image_hw_to_feature_size = {
24 (1008, 1344): 1921,
25 + (2016, 2688): 1933,
26 }
27
28 for (h, w), f in image_hw_to_feature_size.items():
@@ -75,6 +76,9 @@ if is_cpu():
76 # TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
77 # Since we use _attn_implementation="eager" for hf_runner, here is
78 # numeric difference for longer context and test can't pass
79 +@pytest.mark.xfail(
80 + reason="Inconsistent image processor being used due to lack "
81 + "of support for dynamic image token replacement")
82 @pytest.mark.parametrize("model_and_config", model_and_vl_config)
83 @pytest.mark.parametrize("dtype", [target_dtype])
84 @pytest.mark.parametrize("max_tokens", [128])
vllm/model_executor/models/phi3v.py
+65 -4
@@ -13,14 +13,17 @@
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
16 +from typing import Dict, Iterable, List, Literal, Optional, Tuple, TypedDict
17
18 +import numpy as np
19 import torch
20 import torch.nn as nn
21 +from PIL import Image
22 from transformers import CLIPVisionConfig, PretrainedConfig
23
24 from vllm.attention import AttentionMetadata
23 -from vllm.config import CacheConfig, VisionLanguageConfig
25 +from vllm.config import CacheConfig, ModelConfig, VisionLanguageConfig
26 +from vllm.logger import init_logger
27 from vllm.model_executor.layers.logits_processor import LogitsProcessor
28 from vllm.model_executor.layers.quantization.base_config import (
29 QuantizationConfig)
@@ -32,9 +35,11 @@ from vllm.model_executor.models.llama import LlamaModel
35 from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
37 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.image import get_dummy_image_data
38 +from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
39 from vllm.sequence import SamplerOutput
40
41 +logger = init_logger(__name__)
42 +
43 _KEYS_TO_MODIFY_MAPPING = {
44 "model.vision_embed_tokens": "vision_embed_tokens",
45 }
@@ -268,7 +273,63 @@ class Phi3VImagePixelInputs(TypedDict):
273 """Shape: (batch_size, 2)"""
274
275
271 -@MULTIMODAL_REGISTRY.register_image_pixel_input()
276 +# FIXME(Isotr0py): Remove these after dynamic num_img_tokens is supported
277 +# copied from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
278 +def calc_padded_size(width, height, padding_unit=336):
279 + target_height = int(np.ceil(height / padding_unit) * padding_unit)
280 + top_padding = int((target_height - height) / 2)
281 + bottom_padding = target_height - height - top_padding
282 + padded_width = width
283 + padded_height = height + top_padding + bottom_padding
284 + return padded_width, padded_height
285 +
286 +
287 +# copied from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
288 +def calc_hd_transform_size(width, height, hd_num=16):
289 + transposed = False
290 + if width < height:
291 + width, height = height, width
292 + transposed = True
293 +
294 + ratio = width / height
295 + scale = 1
296 + while scale * np.ceil(scale / ratio) <= hd_num:
297 + scale += 1
298 + scale -= 1
299 +
300 + new_width = int(scale * 336)
301 + new_height = int(new_width / ratio)
302 +
303 + padded_width, padded_height = calc_padded_size(new_width, new_height)
304 +
305 + if transposed:
306 + padded_width, padded_height = padded_height, padded_width
307 +
308 + return padded_width, padded_height
309 +
310 +
311 +def _image_processor(
312 + data: ImagePixelData,
313 + model_config: ModelConfig,
314 + vlm_config: VisionLanguageConfig,
315 +) -> Dict[str, torch.Tensor]:
316 + image = data.image
317 +
318 + if isinstance(image, Image.Image):
319 + # Temporary patch before dynamic number of image tokens is supported
320 + _, _, h, w = vlm_config.image_input_shape
321 + if (w, h) != calc_hd_transform_size(image.width, image.height):
322 + logger.warning(
323 + "Dynamic image shape is currently not supported. "
324 + "Resizing input image to (%d, %d).", w, h)
325 +
326 + data.image = image.resize((w, h))
327 +
328 + return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
329 + ._default_input_processor(data, model_config, vlm_config)
330 +
331 +
332 +@MULTIMODAL_REGISTRY.register_image_pixel_input(_image_processor)
333 @MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
334 class Phi3VForCausalLM(VisionLanguageModelBase):
335