@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51a624bf02
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Move some multimodal utils to modality-specific modules (#11494)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 26, 2024 at 12:23 UTC 51a624bf024e351e678b598521b72a2e19b5e2ef
13 files changed +84 -77
tests/models/decoder_only/vision_language/test_awq.py
+1 -1
@@ -3,7 +3,7 @@ from typing import List, Optional, Type
3 import pytest
4 import torch
5
6 -from vllm.multimodal.utils import rescale_image_size
6 +from vllm.multimodal.image import rescale_image_size
7
8 from ....conftest import IMAGE_ASSETS, VllmRunner, _ImageAssets
9 from ...utils import check_logprobs_close
tests/models/decoder_only/vision_language/test_h2ovl.py
+1 -1
@@ -8,7 +8,7 @@ from transformers import AutoConfig
8 # Import the functions to test
9 from vllm.model_executor.models.h2ovl import (calculate_num_blocks,
10 image_to_pixel_values_wrapper)
11 -from vllm.multimodal.utils import rescale_image_size
11 +from vllm.multimodal.image import rescale_image_size
12
13 models = [
14 "h2oai/h2ovl-mississippi-800m", # Replace with your actual model names
tests/models/decoder_only/vision_language/test_phi3v.py
+1 -1
@@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, Type
5 import pytest
6 from transformers import AutoTokenizer
7
8 -from vllm.multimodal.utils import rescale_image_size
8 +from vllm.multimodal.image import rescale_image_size
9 from vllm.platforms import current_platform
10 from vllm.sequence import SampleLogprobs
11
tests/models/decoder_only/vision_language/test_qwen2_vl.py
+2 -2
@@ -6,8 +6,8 @@ import torch
6 from PIL import Image
7
8 from vllm.entrypoints.llm import LLM
9 -from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
10 - sample_frames_from_video)
9 +from vllm.multimodal.image import rescale_image_size
10 +from vllm.multimodal.video import rescale_video_size, sample_frames_from_video
11
12 from ....conftest import (IMAGE_ASSETS, VIDEO_ASSETS, PromptImageInput,
13 PromptVideoInput, VllmRunner)
tests/models/decoder_only/vision_language/vlm_utils/builders.py
+3 -2
@@ -5,8 +5,9 @@ from typing import Callable, Iterable, List, Optional, Tuple, Union
5
6 import torch
7
8 -from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
9 - resize_video, sample_frames_from_video)
8 +from vllm.multimodal.image import rescale_image_size
9 +from vllm.multimodal.video import (rescale_video_size, resize_video,
10 + sample_frames_from_video)
11
12 from .....conftest import _ImageAssets, _VideoAssets
13 from .types import (SINGLE_IMAGE_BASE_PROMPTS, TEST_IMG_PLACEHOLDER,
tests/models/decoder_only/vision_language/vlm_utils/custom_inputs.py
+3 -2
@@ -1,8 +1,9 @@
1 """Custom input builders for edge-cases in different models."""
2 from typing import Callable
3
4 -from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
5 - resize_video, sample_frames_from_video)
4 +from vllm.multimodal.image import rescale_image_size
5 +from vllm.multimodal.video import (rescale_video_size, resize_video,
6 + sample_frames_from_video)
7
8 from .....conftest import IMAGE_ASSETS, VIDEO_ASSETS
9 from .builders import build_multi_image_inputs, build_single_image_inputs
tests/models/encoder_decoder/vision_language/test_mllama.py
+1 -1
@@ -6,7 +6,7 @@ from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6
7 from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
8 global_force_attn_backend_context_manager)
9 -from vllm.multimodal.utils import rescale_image_size
9 +from vllm.multimodal.image import rescale_image_size
10 from vllm.sequence import SampleLogprobs
11
12 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
tests/multimodal/test_mapper.py
+1 -1
@@ -6,7 +6,7 @@ from transformers import LlavaNextImageProcessor
6
7 from vllm.config import ModelConfig
8 from vllm.multimodal import MultiModalRegistry
9 -from vllm.multimodal.utils import rescale_image_size
9 +from vllm.multimodal.image import rescale_image_size
10
11
12 @pytest.fixture
vllm/assets/video.py
+1 -1
@@ -7,7 +7,7 @@ import numpy.typing as npt
7 from huggingface_hub import hf_hub_download
8 from PIL import Image
9
10 -from vllm.multimodal.utils import (sample_frames_from_video,
10 +from vllm.multimodal.video import (sample_frames_from_video,
11 try_import_video_packages)
12
13 from .base import get_cache_dir
vllm/multimodal/audio.py
+12
@@ -1,3 +1,5 @@
1 +from typing import Any
2 +
3 import numpy as np
4 import numpy.typing as npt
5
@@ -26,6 +28,16 @@ class AudioPlugin(MultiModalPlugin):
28 "There is no default maximum multimodal tokens")
29
30
31 +def try_import_audio_packages() -> tuple[Any, Any]:
32 + try:
33 + import librosa
34 + import soundfile
35 + except ImportError as exc:
36 + raise ImportError(
37 + "Please install vllm[audio] for audio support.") from exc
38 + return librosa, soundfile
39 +
40 +
41 def resample_audio(
42 audio: npt.NDArray[np.floating],
43 *,
vllm/multimodal/image.py
+12
@@ -84,3 +84,15 @@ class ImagePlugin(MultiModalPlugin):
84
85 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
86 return 3000
87 +
88 +
89 +def rescale_image_size(image: Image.Image,
90 + size_factor: float,
91 + transpose: int = -1) -> Image.Image:
92 + """Rescale the dimensions of an image by a constant factor."""
93 + new_width = int(image.width * size_factor)
94 + new_height = int(image.height * size_factor)
95 + image = image.resize((new_width, new_height))
96 + if transpose >= 0:
97 + image = image.transpose(Image.Transpose(transpose))
98 + return image
vllm/multimodal/utils.py
+3 -65
@@ -2,7 +2,7 @@ import base64
2 import os
3 from functools import lru_cache
4 from io import BytesIO
5 -from typing import Any, List, Optional, Tuple, TypeVar, Union
5 +from typing import List, Optional, Tuple, TypeVar, Union
6
7 import numpy as np
8 import numpy.typing as npt
@@ -14,7 +14,9 @@ from vllm.connections import global_http_connection
14 from vllm.logger import init_logger
15 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
16
17 +from .audio import try_import_audio_packages
18 from .inputs import MultiModalDataDict, PlaceholderRange
19 +from .video import try_import_video_packages
20
21 logger = init_logger(__name__)
22
@@ -198,16 +200,6 @@ async def async_fetch_video(video_url: str,
200 return video
201
202
201 -def try_import_audio_packages() -> Tuple[Any, Any]:
202 - try:
203 - import librosa
204 - import soundfile
205 - except ImportError as exc:
206 - raise ImportError(
207 - "Please install vllm[audio] for audio support.") from exc
208 - return librosa, soundfile
209 -
210 -
203 def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
204 """
205 Load audio from a URL.
@@ -324,60 +316,6 @@ def load_image_from_base64(image: Union[bytes, str]) -> Image.Image:
316 return _load_image_from_bytes(base64.b64decode(image))
317
318
327 -def rescale_image_size(image: Image.Image,
328 - size_factor: float,
329 - transpose: int = -1) -> Image.Image:
330 - """Rescale the dimensions of an image by a constant factor."""
331 - new_width = int(image.width * size_factor)
332 - new_height = int(image.height * size_factor)
333 - image = image.resize((new_width, new_height))
334 - if transpose >= 0:
335 - image = image.transpose(Image.Transpose(transpose))
336 - return image
337 -
338 -
339 -def try_import_video_packages():
340 - try:
341 - import cv2
342 - import decord
343 - except ImportError as exc:
344 - raise ImportError(
345 - "Please install vllm[video] for video support.") from exc
346 - return cv2, decord
347 -
348 -
349 -def resize_video(frames: npt.NDArray, size: Tuple[int, int]) -> npt.NDArray:
350 - cv2, _ = try_import_video_packages()
351 -
352 - num_frames, _, _, channels = frames.shape
353 - new_height, new_width = size
354 - resized_frames = np.empty((num_frames, new_height, new_width, channels),
355 - dtype=frames.dtype)
356 - for i, frame in enumerate(frames):
357 - resized_frame = cv2.resize(frame, (new_width, new_height))
358 - resized_frames[i] = resized_frame
359 - return resized_frames
360 -
361 -
362 -def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray:
363 - _, height, width, _ = frames.shape
364 - new_height = int(height * size_factor)
365 - new_width = int(width * size_factor)
366 -
367 - return resize_video(frames, (new_height, new_width))
368 -
369 -
370 -def sample_frames_from_video(frames: npt.NDArray,
371 - num_frames: int) -> npt.NDArray:
372 - total_frames = frames.shape[0]
373 - if num_frames == -1:
374 - return frames
375 - else:
376 - frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
377 - sampled_frames = frames[frame_indices, ...]
378 - return sampled_frames
379 -
380 -
319 def encode_video_base64(frames: npt.NDArray) -> str:
320 base64_frames = []
321 frames_list = [frames[i] for i in range(frames.shape[0])]
vllm/multimodal/video.py
+43
@@ -2,6 +2,7 @@ from functools import lru_cache
2 from typing import TYPE_CHECKING, Any, Dict, Optional
3
4 import numpy as np
5 +import numpy.typing as npt
6
7 from vllm.inputs.registry import InputContext
8 from vllm.logger import init_logger
@@ -75,3 +76,45 @@ class VideoPlugin(ImagePlugin):
76
77 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
78 return 4096
79 +
80 +
81 +def try_import_video_packages() -> tuple[Any, Any]:
82 + try:
83 + import cv2
84 + import decord
85 + except ImportError as exc:
86 + raise ImportError(
87 + "Please install vllm[video] for video support.") from exc
88 + return cv2, decord
89 +
90 +
91 +def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
92 + cv2, _ = try_import_video_packages()
93 +
94 + num_frames, _, _, channels = frames.shape
95 + new_height, new_width = size
96 + resized_frames = np.empty((num_frames, new_height, new_width, channels),
97 + dtype=frames.dtype)
98 + for i, frame in enumerate(frames):
99 + resized_frame = cv2.resize(frame, (new_width, new_height))
100 + resized_frames[i] = resized_frame
101 + return resized_frames
102 +
103 +
104 +def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray:
105 + _, height, width, _ = frames.shape
106 + new_height = int(height * size_factor)
107 + new_width = int(width * size_factor)
108 +
109 + return resize_video(frames, (new_height, new_width))
110 +
111 +
112 +def sample_frames_from_video(frames: npt.NDArray,
113 + num_frames: int) -> npt.NDArray:
114 + total_frames = frames.shape[0]
115 + if num_frames == -1:
116 + return frames
117 +
118 + frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
119 + sampled_frames = frames[frame_indices, ...]
120 + return sampled_frames