@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f8d8075f9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model][VLM] Add multi-video support for LLaVA-Onevision (#8905)

Co-authored-by: litianjian <litianjian@bytedance.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

litianjian committed Oct 29, 2024 at 02:04 UTC 5f8d8075f957d5376b2f1cc451e35a2a757e95a5
5 files changed +123 -162
tests/models/decoder_only/vision_language/test_llava_onevision.py
+48 -125
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple, Type, overload
1 +from typing import List, Optional, Tuple, Type
2
3 import pytest
4 from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
@@ -9,9 +9,8 @@ from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
9 from vllm.sequence import SampleLogprobs
10 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
12 -from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13 - _VideoAssets)
14 -from ....utils import large_gpu_test
12 +from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput,
13 + PromptVideoInput, VllmRunner)
14 from ...utils import check_logprobs_close
15
16 # Video test
@@ -20,7 +19,7 @@ HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
19 "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n" # noqa: E501
20 })
21
23 -models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
22 +models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
23
24
25 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -47,50 +46,16 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
46 return hf_output_ids, hf_output_str, out_logprobs
47
48
50 -@overload
51 -def run_video_test(
52 - hf_runner: Type[HfRunner],
53 - vllm_runner: Type[VllmRunner],
54 - video_assets: _VideoAssets,
55 - model: str,
56 - *,
57 - size_factors: List[float],
58 - dtype: str,
59 - max_tokens: int,
60 - num_logprobs: int,
61 - num_frames: int,
62 - tensor_parallel_size: int,
63 - distributed_executor_backend: Optional[str] = None,
64 -):
65 - ...
66 -
67 -
68 -@overload
69 -def run_video_test(
70 - hf_runner: Type[HfRunner],
71 - vllm_runner: Type[VllmRunner],
72 - video_assets: _VideoAssets,
73 - model: str,
74 - *,
75 - sizes: List[Tuple[int, int]],
76 - dtype: str,
77 - max_tokens: int,
78 - num_logprobs: int,
79 - num_frames: int,
80 - tensor_parallel_size: int,
81 - distributed_executor_backend: Optional[str] = None,
82 -):
83 - ...
49 +# Video test
50 +_LIMIT_VIDEO_PER_PROMPT = 4
51
52
53 def run_video_test(
54 hf_runner: Type[HfRunner],
55 vllm_runner: Type[VllmRunner],
89 - video_assets: _VideoAssets,
56 + inputs: List[Tuple[List[str], PromptVideoInput]],
57 model: str,
58 *,
92 - size_factors: Optional[List[float]] = None,
93 - sizes: Optional[List[Tuple[int, int]]] = None,
59 dtype: str,
60 max_tokens: int,
61 num_logprobs: int,
@@ -99,38 +64,20 @@ def run_video_test(
64 distributed_executor_backend: Optional[str] = None,
65 ):
66 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
102 -
103 - videos = [
104 - sample_frames_from_video(asset.np_ndarrays, num_frames)
105 - for asset in video_assets
106 - ]
107 -
108 - if size_factors is not None:
109 - inputs_per_video = [(
110 - [prompt for _ in size_factors],
111 - [rescale_video_size(video, factor) for factor in size_factors],
112 - ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
113 - elif sizes is not None:
114 - inputs_per_video = [(
115 - [prompt for _ in sizes],
116 - [resize_video(video, size) for size in sizes],
117 - ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
118 - else:
119 - raise ValueError("You must provide either `size_factors` or `sizes`")
120 -
121 - # max_model_len should be greater than image_feature_size
67 with vllm_runner(model,
68 dtype=dtype,
124 - max_model_len=4096,
69 + max_model_len=16384,
70 tensor_parallel_size=tensor_parallel_size,
71 distributed_executor_backend=distributed_executor_backend,
127 - enforce_eager=True) as vllm_model:
128 - vllm_outputs_per_video = [
72 + enforce_eager=True,
73 + limit_mm_per_prompt={"video": _LIMIT_VIDEO_PER_PROMPT
74 + }) as vllm_model:
75 + vllm_outputs_per_input = [
76 vllm_model.generate_greedy_logprobs(prompts,
77 max_tokens,
78 num_logprobs=num_logprobs,
79 videos=videos)
133 - for prompts, videos in inputs_per_video
80 + for prompts, videos in inputs
81 ]
82
83 def process(hf_inputs: BatchEncoding):
@@ -142,16 +89,16 @@ def run_video_test(
89 dtype=dtype,
90 postprocess_inputs=process,
91 auto_cls=AutoModelForVision2Seq) as hf_model:
145 - hf_outputs_per_video = [
92 + hf_outputs_per_input = [
93 hf_model.generate_greedy_logprobs_limit(prompts,
94 max_tokens,
95 num_logprobs=num_logprobs,
96 videos=videos)
150 - for prompts, videos in inputs_per_video
97 + for prompts, videos in inputs
98 ]
99
153 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_video,
154 - vllm_outputs_per_video):
100 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_input,
101 + vllm_outputs_per_input):
102 # TODO: Check whether using original CLIPVisionModel can improve
103 # consistency against HF
104 check_logprobs_close(
@@ -165,74 +112,51 @@ def run_video_test(
112 )
113
114
168 -@large_gpu_test(min_gb=48)
115 @pytest.mark.parametrize("model", models)
170 -@pytest.mark.parametrize(
171 - "size_factors",
172 - [
173 - # No video
174 - [],
175 - # Single-scale
176 - [1.0],
177 - # Single-scale, batched
178 - [1.0, 1.0, 1.0],
179 - # Multi-scale
180 - [0.25, 0.5, 1.0],
181 - ],
182 -)
116 @pytest.mark.parametrize("dtype", ["half"])
117 @pytest.mark.parametrize("max_tokens", [128])
118 @pytest.mark.parametrize("num_logprobs", [5])
119 @pytest.mark.parametrize("num_frames", [16])
187 -def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
188 - dtype, max_tokens, num_logprobs, num_frames) -> None:
189 - """Inference result should be the same between hf and vllm.
190 -
191 - All the image fixtures for the test is under tests/videos.
192 - For huggingface runner, we provide the np.ndarray as input.
193 - For vllm runner, we provide MultiModalDataDict objects
194 - and corresponding MultiModalConfig as input.
195 - Note, the text input is also adjusted to abide by vllm contract.
196 - The text output is sanitized to be able to compare with hf.
197 - """
120 +def test_models_multiple_video_inputs(hf_runner, vllm_runner, video_assets,
121 + model, dtype, max_tokens, num_logprobs,
122 + num_frames) -> None:
123 + video = sample_frames_from_video(video_assets[0].np_ndarrays, num_frames)
124 + inputs = [(
125 + [
126 + "<|im_start|>user <video><video>\nDescribe 2 videos. \
127 + <|im_end|><|im_start|>assistant\n",
128 + "<|im_start|>user <video><video>\nDescribe 2 videos. \
129 + <|im_end|><|im_start|>assistant\n",
130 + "<|im_start|>user <video><video><video><video>\nDescribe 4 videos. \
131 + <|im_end|><|im_start|>assistant\n",
132 + "<|im_start|>user <video>\nwhy is this video funny? \
133 + <|im_end|><|im_start|>assistant\n",
134 + ],
135 + [
136 + [video, video],
137 + # Images with different sizes and aspect-ratios
138 + [
139 + rescale_video_size(video, 0.1),
140 + video,
141 + ],
142 + [
143 + video,
144 + rescale_video_size(video, 0.25),
145 + resize_video(video, (183, 488)),
146 + resize_video(video, (488, 183))
147 + ],
148 + video,
149 + ])]
150 run_video_test(
151 hf_runner,
152 vllm_runner,
201 - video_assets,
153 + inputs,
154 model,
203 - size_factors=size_factors,
155 dtype=dtype,
156 max_tokens=max_tokens,
157 num_logprobs=num_logprobs,
207 - num_frames=num_frames,
158 tensor_parallel_size=1,
209 - )
210 -
211 -
212 -@large_gpu_test(min_gb=48)
213 -@pytest.mark.parametrize("model", models)
214 -@pytest.mark.parametrize(
215 - "sizes",
216 - [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
217 -)
218 -@pytest.mark.parametrize("dtype", ["half"])
219 -@pytest.mark.parametrize("max_tokens", [128])
220 -@pytest.mark.parametrize("num_logprobs", [5])
221 -@pytest.mark.parametrize("num_frames", [16])
222 -def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
223 - dtype, max_tokens, num_logprobs,
224 - num_frames) -> None:
225 - run_video_test(
226 - hf_runner,
227 - vllm_runner,
228 - video_assets,
229 - model,
230 - sizes=sizes,
231 - dtype=dtype,
232 - max_tokens=max_tokens,
233 - num_logprobs=num_logprobs,
159 num_frames=num_frames,
235 - tensor_parallel_size=1,
160 )
161
162
@@ -303,7 +227,6 @@ def run_image_test(
227 )
228
229
306 -@large_gpu_test(min_gb=48)
230 @pytest.mark.parametrize("model", models)
231 @pytest.mark.parametrize("dtype", ["half"])
232 @pytest.mark.parametrize("max_tokens", [128])
vllm/model_executor/models/clip.py
+3 -1
@@ -88,6 +88,7 @@ def dummy_image_for_clip(
88 def dummy_video_for_clip(
89 hf_config: CLIPVisionConfig,
90 num_frames: int,
91 + num_videos: int = 1,
92 *,
93 image_width_override: Optional[int] = None,
94 image_height_override: Optional[int] = None,
@@ -99,7 +100,8 @@ def dummy_video_for_clip(
100 image_height_override=image_height_override)
101 np_frame = np.array(pil_frame["image"])
102 mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
102 - mm_data = {"video": mm_data_per_video}
103 + video_data = [mm_data_per_video] * num_videos
104 + mm_data = {"video": video_data}
105 return mm_data
106
107
vllm/model_executor/models/llava_onevision.py
+66 -28
@@ -43,19 +43,17 @@ MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
43
44 # For profile run
45 _MAX_FRAMES_PER_VIDEO = 16
46 -_MAX_NUM_VIDEOS = 1
46
47
48 class LlavaOnevisionVideoPixelInputs(TypedDict):
49 type: Literal["pixel_values_videos"]
50 data: Union[torch.Tensor, List[torch.Tensor]]
51 """
53 - Shape: `(batch_size, num_frames, num_channels, height, width)`
52 + Shape: `(batch_size, num_videos, num_frames, num_channels, height, width)`
53
55 - Note that `num_frames` may be different for each batch, in which case
56 - the data is passed as a list instead of a batched tensor.
57 -
58 - Note that it only supports one video input for one batch.
54 + Note that `num_videos` may be different for each batch, and 'num_frames'
55 + may be different for each video, in which case the data is passed as a
56 + list instead of a batched tensor.
57 """
58
59
@@ -213,11 +211,7 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
211 hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
212 vision_config = hf_config.vision_config
213
216 - # TODO: support multiple videos
214 num_videos = mm_counts["video"]
218 - if num_videos > _MAX_NUM_VIDEOS:
219 - raise NotImplementedError(
220 - f"Only {_MAX_NUM_VIDEOS} videos are supported")
215
216 # TODO: support configuring the number of frames
217 num_frames = _MAX_FRAMES_PER_VIDEO
@@ -232,7 +226,9 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
226 image_feature_size_override=video_feature_size,
227 )
228
235 - mm_data = dummy_video_for_clip(vision_config, num_frames=num_frames)
229 + mm_data = dummy_video_for_clip(vision_config,
230 + num_frames=num_frames,
231 + num_videos=num_videos)
232 return seq_data, mm_data
233 elif isinstance(vision_config, SiglipVisionConfig):
234 seq_data = dummy_seq_data_for_siglip(
@@ -243,7 +239,9 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
239 image_feature_size_override=video_feature_size,
240 )
241
246 - mm_data = dummy_video_for_siglip(vision_config, num_frames=num_frames)
242 + mm_data = dummy_video_for_siglip(vision_config,
243 + num_frames=num_frames,
244 + num_videos=num_videos)
245 return seq_data, mm_data
246
247 msg = f"Unsupported vision config: {type(vision_config)}"
@@ -315,7 +313,6 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
313
314 model_config = ctx.model_config
315 hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
318 - vision_config = hf_config.vision_config
316
317 if isinstance(video_data, np.ndarray):
318 # Supports both CLIP and Siglip
@@ -336,10 +333,27 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
333 multi_modal_data=multi_modal_data)
334
335 elif is_list_of(video_data, np.ndarray):
339 - raise NotImplementedError(
340 - "Processing multiple videos is not supported")
336 + video_feature_size = []
337 + for video in video_data:
338 + num_frames = video.shape[0]
339 + video_feature_size.append(
340 + get_llava_onevision_video_tokens(ctx, num_frames))
341
342 - msg = f"Unsupported vision config: {type(vision_config)}"
342 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
343 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
344 + tokenizer,
345 + inputs.get("prompt"),
346 + inputs["prompt_token_ids"],
347 + placeholder_token_id=hf_config.video_token_index,
348 + repeat_count=video_feature_size,
349 + )
350 + return token_inputs(prompt_token_ids=new_token_ids,
351 + prompt=new_prompt,
352 + multi_modal_data=multi_modal_data)
353 + else:
354 + raise TypeError(f"Invalid video type: {type(video_data)}")
355 +
356 + msg = f"Unsupported video type: {type(video_data)}"
357 raise NotImplementedError(msg)
358
359
@@ -723,6 +737,22 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
737 for i, patch_features_batch in enumerate(patch_embeddings)
738 ]
739
740 + def _add_image_newline(
741 + self,
742 + video_features: torch.Tensor,
743 + videos: int = 1,
744 + frames: int = 1,
745 + strategy: str = "one_token",
746 + ) -> torch.Tensor:
747 + if strategy == "one_token":
748 + video_features = video_features.reshape(
749 + videos, frames * video_features.shape[1], -1)
750 + image_newline = self.image_newline[None, None, :].repeat(
751 + videos, 1, 1).to(video_features.device)
752 + video_features = torch.cat((video_features, image_newline), dim=1)
753 + return video_features
754 + raise ValueError(f"Unexpected video newline strategy: {strategy}")
755 +
756 def _video_pixels_to_features(
757 self,
758 vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
@@ -731,9 +761,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
761
762 # NOTE: we skip the step to select the vision feature layer since
763 # this is already done inside the vision tower
734 - b, num_videos, frames, c, h, w = pixel_values.shape
735 - assert (num_videos == _MAX_NUM_VIDEOS)
736 - pixel_values = pixel_values.reshape(b * num_videos * frames, c, h, w)
764 video_features = vision_tower(pixel_values)
765 video_features = self._select_image_features(
766 video_features,
@@ -741,13 +768,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
768 )
769 video_features = self.multi_modal_projector(video_features)
770 video_features = self.apply_pooling(video_features)
744 - video_features = video_features.reshape(
745 - b, frames * video_features.shape[1], -1)
746 - image_newline = self.image_newline[None, None, :].repeat(b, 1, 1).to(
747 - video_features.device)
748 - video_features = torch.cat((video_features, image_newline), dim=1)
749 - video_features = video_features.flatten(0, 1)
750 -
771 return video_features
772
773 def _process_video_pixels(self, inputs: LlavaOnevisionVideoPixelInputs):
@@ -755,10 +775,28 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
775
776 video_pixels = inputs["data"]
777
758 - # TODO: support multiple videos per input
778 if isinstance(video_pixels, torch.Tensor):
779 + b, num_videos, frames, c, h, w = video_pixels.shape
780 + pixel_values = video_pixels.view(b * num_videos * frames, c, h, w)
781 stacked_embeddings = self._video_pixels_to_features(
761 - self.vision_tower, video_pixels)
782 + self.vision_tower, pixel_values)
783 + stacked_embeddings = self._add_image_newline(stacked_embeddings,
784 + videos=b * num_videos,
785 + frames=frames,
786 + strategy="one_token")
787 + return stacked_embeddings
788 + elif is_list_of(video_pixels, torch.Tensor):
789 + stacked_embeddings = []
790 + for video_pixel in video_pixels:
791 + num_videos, frames, c, h, w = video_pixel.shape
792 + pixel_values = video_pixel.view(num_videos * frames, c, h, w)
793 + embeddings = self._video_pixels_to_features(
794 + self.vision_tower, pixel_values)
795 + embeddings = self._add_image_newline(embeddings,
796 + videos=num_videos,
797 + frames=frames,
798 + strategy="one_token")
799 + stacked_embeddings.append(embeddings)
800 return stacked_embeddings
801 else:
802 raise ValueError(
vllm/model_executor/models/siglip.py
+3 -1
@@ -93,6 +93,7 @@ def dummy_image_for_siglip(
93 def dummy_video_for_siglip(
94 hf_config: SiglipVisionConfig,
95 num_frames: int,
96 + num_videos: int = 1,
97 *,
98 image_width_override: Optional[int] = None,
99 image_height_override: Optional[int] = None,
@@ -104,7 +105,8 @@ def dummy_video_for_siglip(
105 image_height_override=image_height_override)
106 np_frame = np.array(pil_frame["image"])
107 mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
107 - mm_data = {"video": mm_data_per_video}
108 + video_data = [mm_data_per_video] * num_videos
109 + mm_data = {"video": video_data}
110 return mm_data
111
112
vllm/multimodal/video.py
+3 -7
@@ -56,15 +56,14 @@ class VideoPlugin(ImagePlugin):
56 ) -> MultiModalInputs:
57 model_config = ctx.model_config
58
59 - # single video input as np.ndarray
60 - if isinstance(data, np.ndarray):
59 + if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
60 video_processor = self._get_hf_video_processor(
61 model_config,
62 mm_processor_kwargs,
63 )
64 if video_processor is None:
65 raise RuntimeError("No HuggingFace processor is available "
67 - "to process the image object")
66 + "to process the video object")
67 try:
68 # NOTE: Similar to image; it may be a good idea to filter and
69 # pass mm_processor_kwargs here too, but for now we don't to
@@ -72,13 +71,10 @@ class VideoPlugin(ImagePlugin):
71 # signatures of the processor don't align
72 batch_data = video_processor(data, return_tensors="pt").data
73 except Exception:
75 - logger.error("Failed to process image (%s)", data)
74 + logger.error("Failed to process video (%s)", data)
75 raise
76
77 return MultiModalInputs(batch_data)
79 - elif is_list_of(data, np.ndarray):
80 - raise NotImplementedError(
81 - "Multi video for a prompt is not supported yet")
78
79 raise TypeError(f"Invalid video type: {type(data)}")
80