A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model][VLM] Add multi-video support for LLaVA-Onevision (#8905)
Co-authored-by: litianjian <litianjian@bytedance.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
litianjian committed
Oct 29, 2024 at 02:04 UTC
5f8d8075f957d5376b2f1cc451e35a2a757e95a5
5 files changed
+123
-162
tests/models/decoder_only/vision_language/test_llava_onevision.py
+48
-125
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple, Type, overload
1
+from typing import List, Optional, Tuple, Type
2
3
import pytest
4
from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
@@ -9,9 +9,8 @@ from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
9
from vllm.sequence import SampleLogprobs
10
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
12
-from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13
- _VideoAssets)
14
-from ....utils import large_gpu_test
12
+from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput,
13
+ PromptVideoInput, VllmRunner)
14
from ...utils import check_logprobs_close
15
16
# Video test
@@ -20,7 +19,7 @@ HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
19
"<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n" # noqa: E501
20
})
21
23
-models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
22
+models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
23
24
25
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -47,50 +46,16 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
46
return hf_output_ids, hf_output_str, out_logprobs
47
48
50
-@overload
51
-def run_video_test(
52
- hf_runner: Type[HfRunner],
53
- vllm_runner: Type[VllmRunner],
54
- video_assets: _VideoAssets,
55
- model: str,
56
- *,
57
- size_factors: List[float],
58
- dtype: str,
59
- max_tokens: int,
60
- num_logprobs: int,
61
- num_frames: int,
62
- tensor_parallel_size: int,
63
- distributed_executor_backend: Optional[str] = None,
64
-):
65
- ...
66
-
67
-
68
-@overload
69
-def run_video_test(
70
- hf_runner: Type[HfRunner],
71
- vllm_runner: Type[VllmRunner],
72
- video_assets: _VideoAssets,
73
- model: str,
74
- *,
75
- sizes: List[Tuple[int, int]],
76
- dtype: str,
77
- max_tokens: int,
78
- num_logprobs: int,
79
- num_frames: int,
80
- tensor_parallel_size: int,
81
- distributed_executor_backend: Optional[str] = None,
82
-):
83
- ...
49
+# Video test
50
+_LIMIT_VIDEO_PER_PROMPT = 4
51
52
53
def run_video_test(
54
hf_runner: Type[HfRunner],
55
vllm_runner: Type[VllmRunner],
89
- video_assets: _VideoAssets,
56
+ inputs: List[Tuple[List[str], PromptVideoInput]],
57
model: str,
58
*,
92
- size_factors: Optional[List[float]] = None,
93
- sizes: Optional[List[Tuple[int, int]]] = None,
59
dtype: str,
60
max_tokens: int,
61
num_logprobs: int,
@@ -99,38 +64,20 @@ def run_video_test(
64
distributed_executor_backend: Optional[str] = None,
65
):
66
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
102
-
103
- videos = [
104
- sample_frames_from_video(asset.np_ndarrays, num_frames)
105
- for asset in video_assets
106
- ]
107
-
108
- if size_factors is not None:
109
- inputs_per_video = [(
110
- [prompt for _ in size_factors],
111
- [rescale_video_size(video, factor) for factor in size_factors],
112
- ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
113
- elif sizes is not None:
114
- inputs_per_video = [(
115
- [prompt for _ in sizes],
116
- [resize_video(video, size) for size in sizes],
117
- ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
118
- else:
119
- raise ValueError("You must provide either `size_factors` or `sizes`")
120
-
121
- # max_model_len should be greater than image_feature_size
67
with vllm_runner(model,
68
dtype=dtype,
124
- max_model_len=4096,
69
+ max_model_len=16384,
70
tensor_parallel_size=tensor_parallel_size,
71
distributed_executor_backend=distributed_executor_backend,
127
- enforce_eager=True) as vllm_model:
128
- vllm_outputs_per_video = [
72
+ enforce_eager=True,
73
+ limit_mm_per_prompt={"video": _LIMIT_VIDEO_PER_PROMPT
74
+ }) as vllm_model:
75
+ vllm_outputs_per_input = [
76
vllm_model.generate_greedy_logprobs(prompts,
77
max_tokens,
78
num_logprobs=num_logprobs,
79
videos=videos)
133
- for prompts, videos in inputs_per_video
80
+ for prompts, videos in inputs
81
]
82
83
def process(hf_inputs: BatchEncoding):
@@ -142,16 +89,16 @@ def run_video_test(
89
dtype=dtype,
90
postprocess_inputs=process,
91
auto_cls=AutoModelForVision2Seq) as hf_model:
145
- hf_outputs_per_video = [
92
+ hf_outputs_per_input = [
93
hf_model.generate_greedy_logprobs_limit(prompts,
94
max_tokens,
95
num_logprobs=num_logprobs,
96
videos=videos)
150
- for prompts, videos in inputs_per_video
97
+ for prompts, videos in inputs
98
]
99
153
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_video,
154
- vllm_outputs_per_video):
100
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_input,
101
+ vllm_outputs_per_input):
102
# TODO: Check whether using original CLIPVisionModel can improve
103
# consistency against HF
104
check_logprobs_close(
@@ -165,74 +112,51 @@ def run_video_test(
112
)
113
114
168
-@large_gpu_test(min_gb=48)
115
@pytest.mark.parametrize("model", models)
170
-@pytest.mark.parametrize(
171
- "size_factors",
172
- [
173
- # No video
174
- [],
175
- # Single-scale
176
- [1.0],
177
- # Single-scale, batched
178
- [1.0, 1.0, 1.0],
179
- # Multi-scale
180
- [0.25, 0.5, 1.0],
181
- ],
182
-)
116
@pytest.mark.parametrize("dtype", ["half"])
117
@pytest.mark.parametrize("max_tokens", [128])
118
@pytest.mark.parametrize("num_logprobs", [5])
119
@pytest.mark.parametrize("num_frames", [16])
187
-def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
188
- dtype, max_tokens, num_logprobs, num_frames) -> None:
189
- """Inference result should be the same between hf and vllm.
190
-
191
- All the image fixtures for the test is under tests/videos.
192
- For huggingface runner, we provide the np.ndarray as input.
193
- For vllm runner, we provide MultiModalDataDict objects
194
- and corresponding MultiModalConfig as input.
195
- Note, the text input is also adjusted to abide by vllm contract.
196
- The text output is sanitized to be able to compare with hf.
197
- """
120
+def test_models_multiple_video_inputs(hf_runner, vllm_runner, video_assets,
121
+ model, dtype, max_tokens, num_logprobs,
122
+ num_frames) -> None:
123
+ video = sample_frames_from_video(video_assets[0].np_ndarrays, num_frames)
124
+ inputs = [(
125
+ [
126
+ "<|im_start|>user <video><video>\nDescribe 2 videos. \
127
+ <|im_end|><|im_start|>assistant\n",
128
+ "<|im_start|>user <video><video>\nDescribe 2 videos. \
129
+ <|im_end|><|im_start|>assistant\n",
130
+ "<|im_start|>user <video><video><video><video>\nDescribe 4 videos. \
131
+ <|im_end|><|im_start|>assistant\n",
132
+ "<|im_start|>user <video>\nwhy is this video funny? \
133
+ <|im_end|><|im_start|>assistant\n",
134
+ ],
135
+ [
136
+ [video, video],
137
+ # Images with different sizes and aspect-ratios
138
+ [
139
+ rescale_video_size(video, 0.1),
140
+ video,
141
+ ],
142
+ [
143
+ video,
144
+ rescale_video_size(video, 0.25),
145
+ resize_video(video, (183, 488)),
146
+ resize_video(video, (488, 183))
147
+ ],
148
+ video,
149
+ ])]
150
run_video_test(
151
hf_runner,
152
vllm_runner,
201
- video_assets,
153
+ inputs,
154
model,
203
- size_factors=size_factors,
155
dtype=dtype,
156
max_tokens=max_tokens,
157
num_logprobs=num_logprobs,
207
- num_frames=num_frames,
158
tensor_parallel_size=1,
209
- )
210
-
211
-
212
-@large_gpu_test(min_gb=48)
213
-@pytest.mark.parametrize("model", models)
214
-@pytest.mark.parametrize(
215
- "sizes",
216
- [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
217
-)
218
-@pytest.mark.parametrize("dtype", ["half"])
219
-@pytest.mark.parametrize("max_tokens", [128])
220
-@pytest.mark.parametrize("num_logprobs", [5])
221
-@pytest.mark.parametrize("num_frames", [16])
222
-def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
223
- dtype, max_tokens, num_logprobs,
224
- num_frames) -> None:
225
- run_video_test(
226
- hf_runner,
227
- vllm_runner,
228
- video_assets,
229
- model,
230
- sizes=sizes,
231
- dtype=dtype,
232
- max_tokens=max_tokens,
233
- num_logprobs=num_logprobs,
159
num_frames=num_frames,
235
- tensor_parallel_size=1,
160
)
161
162
@@ -303,7 +227,6 @@ def run_image_test(
227
)
228
229
306
-@large_gpu_test(min_gb=48)
230
@pytest.mark.parametrize("model", models)
231
@pytest.mark.parametrize("dtype", ["half"])
232
@pytest.mark.parametrize("max_tokens", [128])
vllm/model_executor/models/clip.py
+3
-1
@@ -88,6 +88,7 @@ def dummy_image_for_clip(
88
def dummy_video_for_clip(
89
hf_config: CLIPVisionConfig,
90
num_frames: int,
91
+ num_videos: int = 1,
92
*,
93
image_width_override: Optional[int] = None,
94
image_height_override: Optional[int] = None,
@@ -99,7 +100,8 @@ def dummy_video_for_clip(
100
image_height_override=image_height_override)
101
np_frame = np.array(pil_frame["image"])
102
mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
102
- mm_data = {"video": mm_data_per_video}
103
+ video_data = [mm_data_per_video] * num_videos
104
+ mm_data = {"video": video_data}
105
return mm_data
106
107
vllm/model_executor/models/llava_onevision.py
+66
-28
@@ -43,19 +43,17 @@ MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
43
44
# For profile run
45
_MAX_FRAMES_PER_VIDEO = 16
46
-_MAX_NUM_VIDEOS = 1
46
47
48
class LlavaOnevisionVideoPixelInputs(TypedDict):
49
type: Literal["pixel_values_videos"]
50
data: Union[torch.Tensor, List[torch.Tensor]]
51
"""
53
- Shape: `(batch_size, num_frames, num_channels, height, width)`
52
+ Shape: `(batch_size, num_videos, num_frames, num_channels, height, width)`
53
55
- Note that `num_frames` may be different for each batch, in which case
56
- the data is passed as a list instead of a batched tensor.
57
-
58
- Note that it only supports one video input for one batch.
54
+ Note that `num_videos` may be different for each batch, and 'num_frames'
55
+ may be different for each video, in which case the data is passed as a
56
+ list instead of a batched tensor.
57
"""
58
59
@@ -213,11 +211,7 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
211
hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
212
vision_config = hf_config.vision_config
213
216
- # TODO: support multiple videos
214
num_videos = mm_counts["video"]
218
- if num_videos > _MAX_NUM_VIDEOS:
219
- raise NotImplementedError(
220
- f"Only {_MAX_NUM_VIDEOS} videos are supported")
215
216
# TODO: support configuring the number of frames
217
num_frames = _MAX_FRAMES_PER_VIDEO
@@ -232,7 +226,9 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
226
image_feature_size_override=video_feature_size,
227
)
228
235
- mm_data = dummy_video_for_clip(vision_config, num_frames=num_frames)
229
+ mm_data = dummy_video_for_clip(vision_config,
230
+ num_frames=num_frames,
231
+ num_videos=num_videos)
232
return seq_data, mm_data
233
elif isinstance(vision_config, SiglipVisionConfig):
234
seq_data = dummy_seq_data_for_siglip(
@@ -243,7 +239,9 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
239
image_feature_size_override=video_feature_size,
240
)
241
246
- mm_data = dummy_video_for_siglip(vision_config, num_frames=num_frames)
242
+ mm_data = dummy_video_for_siglip(vision_config,
243
+ num_frames=num_frames,
244
+ num_videos=num_videos)
245
return seq_data, mm_data
246
247
msg = f"Unsupported vision config: {type(vision_config)}"
@@ -315,7 +313,6 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
313
314
model_config = ctx.model_config
315
hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
318
- vision_config = hf_config.vision_config
316
317
if isinstance(video_data, np.ndarray):
318
# Supports both CLIP and Siglip
@@ -336,10 +333,27 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
333
multi_modal_data=multi_modal_data)
334
335
elif is_list_of(video_data, np.ndarray):
339
- raise NotImplementedError(
340
- "Processing multiple videos is not supported")
336
+ video_feature_size = []
337
+ for video in video_data:
338
+ num_frames = video.shape[0]
339
+ video_feature_size.append(
340
+ get_llava_onevision_video_tokens(ctx, num_frames))
341
342
- msg = f"Unsupported vision config: {type(vision_config)}"
342
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
343
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
344
+ tokenizer,
345
+ inputs.get("prompt"),
346
+ inputs["prompt_token_ids"],
347
+ placeholder_token_id=hf_config.video_token_index,
348
+ repeat_count=video_feature_size,
349
+ )
350
+ return token_inputs(prompt_token_ids=new_token_ids,
351
+ prompt=new_prompt,
352
+ multi_modal_data=multi_modal_data)
353
+ else:
354
+ raise TypeError(f"Invalid video type: {type(video_data)}")
355
+
356
+ msg = f"Unsupported video type: {type(video_data)}"
357
raise NotImplementedError(msg)
358
359
@@ -723,6 +737,22 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
737
for i, patch_features_batch in enumerate(patch_embeddings)
738
]
739
740
+ def _add_image_newline(
741
+ self,
742
+ video_features: torch.Tensor,
743
+ videos: int = 1,
744
+ frames: int = 1,
745
+ strategy: str = "one_token",
746
+ ) -> torch.Tensor:
747
+ if strategy == "one_token":
748
+ video_features = video_features.reshape(
749
+ videos, frames * video_features.shape[1], -1)
750
+ image_newline = self.image_newline[None, None, :].repeat(
751
+ videos, 1, 1).to(video_features.device)
752
+ video_features = torch.cat((video_features, image_newline), dim=1)
753
+ return video_features
754
+ raise ValueError(f"Unexpected video newline strategy: {strategy}")
755
+
756
def _video_pixels_to_features(
757
self,
758
vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
@@ -731,9 +761,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
761
762
# NOTE: we skip the step to select the vision feature layer since
763
# this is already done inside the vision tower
734
- b, num_videos, frames, c, h, w = pixel_values.shape
735
- assert (num_videos == _MAX_NUM_VIDEOS)
736
- pixel_values = pixel_values.reshape(b * num_videos * frames, c, h, w)
764
video_features = vision_tower(pixel_values)
765
video_features = self._select_image_features(
766
video_features,
@@ -741,13 +768,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
768
)
769
video_features = self.multi_modal_projector(video_features)
770
video_features = self.apply_pooling(video_features)
744
- video_features = video_features.reshape(
745
- b, frames * video_features.shape[1], -1)
746
- image_newline = self.image_newline[None, None, :].repeat(b, 1, 1).to(
747
- video_features.device)
748
- video_features = torch.cat((video_features, image_newline), dim=1)
749
- video_features = video_features.flatten(0, 1)
750
-
771
return video_features
772
773
def _process_video_pixels(self, inputs: LlavaOnevisionVideoPixelInputs):
@@ -755,10 +775,28 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
775
776
video_pixels = inputs["data"]
777
758
- # TODO: support multiple videos per input
778
if isinstance(video_pixels, torch.Tensor):
779
+ b, num_videos, frames, c, h, w = video_pixels.shape
780
+ pixel_values = video_pixels.view(b * num_videos * frames, c, h, w)
781
stacked_embeddings = self._video_pixels_to_features(
761
- self.vision_tower, video_pixels)
782
+ self.vision_tower, pixel_values)
783
+ stacked_embeddings = self._add_image_newline(stacked_embeddings,
784
+ videos=b * num_videos,
785
+ frames=frames,
786
+ strategy="one_token")
787
+ return stacked_embeddings
788
+ elif is_list_of(video_pixels, torch.Tensor):
789
+ stacked_embeddings = []
790
+ for video_pixel in video_pixels:
791
+ num_videos, frames, c, h, w = video_pixel.shape
792
+ pixel_values = video_pixel.view(num_videos * frames, c, h, w)
793
+ embeddings = self._video_pixels_to_features(
794
+ self.vision_tower, pixel_values)
795
+ embeddings = self._add_image_newline(embeddings,
796
+ videos=num_videos,
797
+ frames=frames,
798
+ strategy="one_token")
799
+ stacked_embeddings.append(embeddings)
800
return stacked_embeddings
801
else:
802
raise ValueError(
vllm/model_executor/models/siglip.py
+3
-1
@@ -93,6 +93,7 @@ def dummy_image_for_siglip(
93
def dummy_video_for_siglip(
94
hf_config: SiglipVisionConfig,
95
num_frames: int,
96
+ num_videos: int = 1,
97
*,
98
image_width_override: Optional[int] = None,
99
image_height_override: Optional[int] = None,
@@ -104,7 +105,8 @@ def dummy_video_for_siglip(
105
image_height_override=image_height_override)
106
np_frame = np.array(pil_frame["image"])
107
mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
107
- mm_data = {"video": mm_data_per_video}
108
+ video_data = [mm_data_per_video] * num_videos
109
+ mm_data = {"video": video_data}
110
return mm_data
111
112
vllm/multimodal/video.py
+3
-7
@@ -56,15 +56,14 @@ class VideoPlugin(ImagePlugin):
56
) -> MultiModalInputs:
57
model_config = ctx.model_config
58
59
- # single video input as np.ndarray
60
- if isinstance(data, np.ndarray):
59
+ if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
60
video_processor = self._get_hf_video_processor(
61
model_config,
62
mm_processor_kwargs,
63
)
64
if video_processor is None:
65
raise RuntimeError("No HuggingFace processor is available "
67
- "to process the image object")
66
+ "to process the video object")
67
try:
68
# NOTE: Similar to image; it may be a good idea to filter and
69
# pass mm_processor_kwargs here too, but for now we don't to
@@ -72,13 +71,10 @@ class VideoPlugin(ImagePlugin):
71
# signatures of the processor don't align
72
batch_data = video_processor(data, return_tensors="pt").data
73
except Exception:
75
- logger.error("Failed to process image (%s)", data)
74
+ logger.error("Failed to process video (%s)", data)
75
raise
76
77
return MultiModalInputs(batch_data)
79
- elif is_list_of(data, np.ndarray):
80
- raise NotImplementedError(
81
- "Multi video for a prompt is not supported yet")
78
79
raise TypeError(f"Invalid video type: {type(data)}")
80