A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Add phi3v resize for dynamic shape and fix torchvision requirement (#5772)
Isotr0py committed
Jun 24, 2024 at 12:11 UTC
edd5fe5fa29b8f9cc5fa37a30cc7211e0ff37067
5 files changed
+72
-5
requirements-cpu.txt
+1
@@ -3,4 +3,5 @@
3
4
# Dependencies for x86_64 CPUs
5
torch == 2.3.1+cpu
6
+torchvision == 0.18.1+cpu # required for the image processor of phi3v, this must be updated alongside torch
7
triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
\ No newline at end of file
requirements-cuda.txt
+2
@@ -5,5 +5,7 @@
5
ray >= 2.9
6
nvidia-ml-py # for pynvml package
7
torch == 2.3.0
8
+# These must be updated alongside torch
9
+torchvision == 0.18.0 # Required for phi3v processor, also see https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
11
vllm-flash-attn == 2.5.9 # Requires PyTorch 2.3.0
requirements-test.txt
-1
@@ -14,7 +14,6 @@ peft
14
requests
15
ray
16
sentence-transformers # required for embedding
17
-torchvision # required for the image processor of phi3v
17
18
# Benchmarking
19
aiohttp
tests/models/test_phi3v.py
+4
@@ -22,6 +22,7 @@ assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
22
def iter_phi3v_configs(model_name: str):
23
image_hw_to_feature_size = {
24
(1008, 1344): 1921,
25
+ (2016, 2688): 1933,
26
}
27
28
for (h, w), f in image_hw_to_feature_size.items():
@@ -75,6 +76,9 @@ if is_cpu():
76
# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
77
# Since we use _attn_implementation="eager" for hf_runner, here is
78
# numeric difference for longer context and test can't pass
79
+@pytest.mark.xfail(
80
+ reason="Inconsistent image processor being used due to lack "
81
+ "of support for dynamic image token replacement")
82
@pytest.mark.parametrize("model_and_config", model_and_vl_config)
83
@pytest.mark.parametrize("dtype", [target_dtype])
84
@pytest.mark.parametrize("max_tokens", [128])
vllm/model_executor/models/phi3v.py
+65
-4
@@ -13,14 +13,17 @@
13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
# See the License for the specific language governing permissions and
15
# limitations under the License.
16
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
16
+from typing import Dict, Iterable, List, Literal, Optional, Tuple, TypedDict
17
18
+import numpy as np
19
import torch
20
import torch.nn as nn
21
+from PIL import Image
22
from transformers import CLIPVisionConfig, PretrainedConfig
23
24
from vllm.attention import AttentionMetadata
23
-from vllm.config import CacheConfig, VisionLanguageConfig
25
+from vllm.config import CacheConfig, ModelConfig, VisionLanguageConfig
26
+from vllm.logger import init_logger
27
from vllm.model_executor.layers.logits_processor import LogitsProcessor
28
from vllm.model_executor.layers.quantization.base_config import (
29
QuantizationConfig)
@@ -32,9 +35,11 @@ from vllm.model_executor.models.llama import LlamaModel
35
from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.multimodal import MULTIMODAL_REGISTRY
35
-from vllm.multimodal.image import get_dummy_image_data
38
+from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
39
from vllm.sequence import SamplerOutput
40
41
+logger = init_logger(__name__)
42
+
43
_KEYS_TO_MODIFY_MAPPING = {
44
"model.vision_embed_tokens": "vision_embed_tokens",
45
}
@@ -268,7 +273,63 @@ class Phi3VImagePixelInputs(TypedDict):
273
"""Shape: (batch_size, 2)"""
274
275
271
-@MULTIMODAL_REGISTRY.register_image_pixel_input()
276
+# FIXME(Isotr0py): Remove these after dynamic num_img_tokens is supported
277
+# copied from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
278
+def calc_padded_size(width, height, padding_unit=336):
279
+ target_height = int(np.ceil(height / padding_unit) * padding_unit)
280
+ top_padding = int((target_height - height) / 2)
281
+ bottom_padding = target_height - height - top_padding
282
+ padded_width = width
283
+ padded_height = height + top_padding + bottom_padding
284
+ return padded_width, padded_height
285
+
286
+
287
+# copied from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
288
+def calc_hd_transform_size(width, height, hd_num=16):
289
+ transposed = False
290
+ if width < height:
291
+ width, height = height, width
292
+ transposed = True
293
+
294
+ ratio = width / height
295
+ scale = 1
296
+ while scale * np.ceil(scale / ratio) <= hd_num:
297
+ scale += 1
298
+ scale -= 1
299
+
300
+ new_width = int(scale * 336)
301
+ new_height = int(new_width / ratio)
302
+
303
+ padded_width, padded_height = calc_padded_size(new_width, new_height)
304
+
305
+ if transposed:
306
+ padded_width, padded_height = padded_height, padded_width
307
+
308
+ return padded_width, padded_height
309
+
310
+
311
+def _image_processor(
312
+ data: ImagePixelData,
313
+ model_config: ModelConfig,
314
+ vlm_config: VisionLanguageConfig,
315
+) -> Dict[str, torch.Tensor]:
316
+ image = data.image
317
+
318
+ if isinstance(image, Image.Image):
319
+ # Temporary patch before dynamic number of image tokens is supported
320
+ _, _, h, w = vlm_config.image_input_shape
321
+ if (w, h) != calc_hd_transform_size(image.width, image.height):
322
+ logger.warning(
323
+ "Dynamic image shape is currently not supported. "
324
+ "Resizing input image to (%d, %d).", w, h)
325
+
326
+ data.image = image.resize((w, h))
327
+
328
+ return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
329
+ ._default_input_processor(data, model_config, vlm_config)
330
+
331
+
332
+@MULTIMODAL_REGISTRY.register_image_pixel_input(_image_processor)
333
@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
334
class Phi3VForCausalLM(VisionLanguageModelBase):
335