A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Add support for Qwen2-VL video embeddings input & multiple image embeddings input with varied resolutions (#10221)
Signed-off-by: imkero <kerorek@outlook.com>
电脑星人 committed
Nov 13, 2024 at 15:07 UTC
3945c82346dae3129213607663bfd17edd905fef
3 files changed
+578
-32
docs/source/models/supported_models.rst
+1
-1
@@ -538,7 +538,7 @@ Text Generation
538
- ✅︎
539
* - :code:`Qwen2VLForConditionalGeneration`
540
- Qwen2-VL
541
- - T + I\ :sup:`E+` + V\ :sup:`+`
541
+ - T + I\ :sup:`E+` + V\ :sup:`E+`
542
- :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
543
- ✅︎
544
- ✅︎
tests/models/decoder_only/vision_language/test_qwen2_vl.py
new
+428
@@ -0,0 +1,428 @@
1
+from typing import Any, List, Optional, Tuple, Type, TypedDict, Union
2
+
3
+import numpy.typing as npt
4
+import pytest
5
+import torch
6
+from PIL import Image
7
+
8
+from vllm.entrypoints.llm import LLM
9
+from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
10
+ sample_frames_from_video)
11
+
12
+from ....conftest import (IMAGE_ASSETS, VIDEO_ASSETS, PromptImageInput,
13
+ PromptVideoInput, VllmRunner)
14
+from ...utils import check_logprobs_close
15
+
16
+models = ["Qwen/Qwen2-VL-2B-Instruct"]
17
+target_dtype = "half"
18
+
19
+IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
20
+VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
21
+
22
+
23
+def qwen2_vl_chat_template(*query):
24
+ return f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{''.join(query)}<|im_end|><|im_start|>assistant\n" # noqa: E501
25
+
26
+
27
+IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
28
+ "stop_sign":
29
+ qwen2_vl_chat_template(
30
+ IMAGE_PLACEHOLDER,
31
+ "What is the biggest text's content in this image?",
32
+ ),
33
+ "cherry_blossom":
34
+ qwen2_vl_chat_template(
35
+ IMAGE_PLACEHOLDER,
36
+ "What is the season shown in this image? ",
37
+ "Reply with a short sentence (no more than 20 words)",
38
+ ),
39
+})
40
+
41
+VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
42
+ "sample_demo_1":
43
+ qwen2_vl_chat_template(
44
+ VIDEO_PLACEHOLDER,
45
+ "Describe this video with a short sentence ",
46
+ "(no more than 20 words)",
47
+ ),
48
+})
49
+
50
+MULTIIMAGE_PROMPT = qwen2_vl_chat_template(
51
+ IMAGE_PLACEHOLDER,
52
+ IMAGE_PLACEHOLDER,
53
+ "Describe these two images separately. ",
54
+ "For each image, reply with a short sentence ",
55
+ "(no more than 10 words).",
56
+)
57
+
58
+
59
+class Qwen2VLPromptImageEmbeddingInput(TypedDict):
60
+ image_embeds: torch.Tensor
61
+ image_grid_thw: torch.Tensor
62
+
63
+
64
+class Qwen2VLPromptVideoEmbeddingInput(TypedDict):
65
+ video_embeds: torch.Tensor
66
+ video_grid_thw: torch.Tensor
67
+
68
+
69
+def batch_make_image_embeddings(
70
+ image_batches: List[Union[Image.Image, List[Image.Image]]], processor,
71
+ llm: LLM) -> List[Qwen2VLPromptImageEmbeddingInput]:
72
+ """batched image embeddings for Qwen2-VL
73
+
74
+ This will infer all images' embeddings in a single batch,
75
+ and split the result according to input batches.
76
+
77
+ image_batches:
78
+ - Single-image batches: `List[Image.Image]`
79
+ - Multiple-image batches: `List[List[Image.Image]]]`
80
+
81
+ returns: `List[Qwen2VLPromptImageEmbeddingInput]`
82
+ """
83
+
84
+ image_batches_: List[Any] = image_batches[:]
85
+
86
+ # convert single-image batches to multiple-image batches
87
+ for idx in range(len(image_batches_)):
88
+ if not isinstance(image_batches_[idx], list):
89
+ image_batches_[idx] = [image_batches_[idx]]
90
+
91
+ assert isinstance(image_batches_[idx], list)
92
+
93
+ # append all images into a list (as a batch)
94
+ images: List[Image.Image] = []
95
+ for image_batch in image_batches_:
96
+ images += image_batch
97
+
98
+ # image to pixel values
99
+ image_processor = processor.image_processor
100
+
101
+ preprocess_result = image_processor \
102
+ .preprocess(images=images, return_tensors="pt") \
103
+ .data
104
+ pixel_values = preprocess_result["pixel_values"]
105
+ image_grid_thw = preprocess_result["image_grid_thw"]
106
+
107
+ # pixel values to embeddinds & grid_thws
108
+ with torch.no_grad():
109
+ visual = llm.llm_engine.model_executor.driver_worker. \
110
+ model_runner.model.visual
111
+
112
+ pixel_values_on_device = pixel_values.to(visual.device,
113
+ dtype=visual.dtype)
114
+ image_grid_thw_on_device = image_grid_thw.to(visual.device,
115
+ dtype=torch.int64)
116
+ image_embeds = visual(pixel_values_on_device,
117
+ grid_thw=image_grid_thw_on_device)
118
+
119
+ # split into original batches
120
+ result: List[Qwen2VLPromptImageEmbeddingInput] = []
121
+ image_counter = 0
122
+ embed_counter = 0
123
+ for image_batch in image_batches_:
124
+ cur_batch_image_count = len(image_batch)
125
+ merge_size = image_processor.merge_size
126
+ cur_batch_embed_len = sum([
127
+ grid_thw.prod() // merge_size // merge_size
128
+ for grid_thw in image_grid_thw[image_counter:image_counter +
129
+ cur_batch_image_count]
130
+ ])
131
+
132
+ result.append({
133
+ "image_embeds":
134
+ image_embeds[embed_counter:embed_counter + cur_batch_embed_len],
135
+ "image_grid_thw":
136
+ image_grid_thw[image_counter:image_counter +
137
+ cur_batch_image_count],
138
+ })
139
+
140
+ embed_counter += cur_batch_embed_len
141
+ image_counter += cur_batch_image_count
142
+
143
+ # ensure we don't lost any images or embeddings
144
+ assert embed_counter == image_embeds.size(0)
145
+ assert image_counter == image_grid_thw.size(0)
146
+ assert len(image_batches) == len(result)
147
+
148
+ return result
149
+
150
+
151
+def batch_make_video_embeddings(
152
+ video_batches: PromptVideoInput, processor,
153
+ llm: LLM) -> List[Qwen2VLPromptVideoEmbeddingInput]:
154
+ """batched video embeddings for Qwen2-VL
155
+
156
+ A NDArray represents a single video's all frames.
157
+
158
+ This will infer all videos' embeddings in a single batch,
159
+ and split the result according to input batches.
160
+
161
+ video_batches:
162
+ - Single-video batches: `List[NDArray]`
163
+ - Multiple-video batches: `List[List[NDArray]]`
164
+ """
165
+
166
+ video_batches_: List[Any] = video_batches[:]
167
+
168
+ for idx in range(len(video_batches_)):
169
+ if not isinstance(video_batches_[idx], list):
170
+ single_video_batch: List[npt.NDArray] = [video_batches_[idx]]
171
+ video_batches_[idx] = single_video_batch
172
+
173
+ assert isinstance(video_batches_[idx], list)
174
+
175
+ # append all videos into a list (as a batch)
176
+ videos: List[npt.NDArray] = []
177
+ for video_batch in video_batches_:
178
+ videos += video_batch
179
+
180
+ # video to pixel values
181
+ image_processor = processor.image_processor
182
+
183
+ preprocess_result = image_processor \
184
+ .preprocess(images=None, videos=videos, return_tensors="pt") \
185
+ .data
186
+ pixel_values = preprocess_result["pixel_values_videos"]
187
+ video_grid_thw = preprocess_result["video_grid_thw"]
188
+
189
+ # pixel values to embeddinds & grid_thws
190
+ with torch.no_grad():
191
+ visual = llm.llm_engine.model_executor.driver_worker.\
192
+ model_runner.model.visual
193
+
194
+ pixel_values_on_device = pixel_values.to(visual.device,
195
+ dtype=visual.dtype)
196
+ video_grid_thw_on_device = video_grid_thw.to(visual.device,
197
+ dtype=torch.int64)
198
+ video_embeds = visual(pixel_values_on_device,
199
+ grid_thw=video_grid_thw_on_device)
200
+
201
+ # split into original batches
202
+ result: List[Qwen2VLPromptVideoEmbeddingInput] = []
203
+ video_counter = 0
204
+ embed_counter = 0
205
+ for video_batch in video_batches_:
206
+ cur_batch_video_count = len(video_batch)
207
+ merge_size = image_processor.merge_size
208
+ cur_batch_embed_len = sum([
209
+ grid_thw.prod() // merge_size // merge_size
210
+ for grid_thw in video_grid_thw[video_counter:video_counter +
211
+ cur_batch_video_count]
212
+ ])
213
+
214
+ result.append({
215
+ "video_embeds":
216
+ video_embeds[embed_counter:embed_counter + cur_batch_embed_len],
217
+ "video_grid_thw":
218
+ video_grid_thw[video_counter:video_counter +
219
+ cur_batch_video_count],
220
+ })
221
+
222
+ embed_counter += cur_batch_embed_len
223
+ video_counter += cur_batch_video_count
224
+
225
+ # ensure we don't lost any videos or embeddings
226
+ assert embed_counter == video_embeds.size(0)
227
+ assert video_counter == video_grid_thw.size(0)
228
+ assert len(video_batches) == len(result)
229
+
230
+ return result
231
+
232
+
233
+def run_test(
234
+ vllm_runner: Type[VllmRunner],
235
+ inputs: List[Tuple[List[str], PromptImageInput, PromptVideoInput]],
236
+ model: str,
237
+ *,
238
+ dtype: str,
239
+ max_tokens: int,
240
+ num_logprobs: int,
241
+ mm_limit: int,
242
+ tensor_parallel_size: int,
243
+ distributed_executor_backend: Optional[str] = None,
244
+):
245
+ """Inference result should be the same between
246
+ original image/video input and image/video embeddings input.
247
+ """
248
+ from transformers import AutoProcessor # noqa: F401
249
+
250
+ processor = AutoProcessor.from_pretrained(model)
251
+
252
+ # NOTE:
253
+ # max_model_len should be greater than image_feature_size
254
+ with vllm_runner(model,
255
+ task="generate",
256
+ max_model_len=4000,
257
+ max_num_seqs=3,
258
+ dtype=dtype,
259
+ limit_mm_per_prompt={
260
+ "image": mm_limit,
261
+ "video": mm_limit
262
+ },
263
+ tensor_parallel_size=tensor_parallel_size,
264
+ distributed_executor_backend=distributed_executor_backend
265
+ ) as vllm_model:
266
+
267
+ outputs_per_case_for_original_input = [
268
+ vllm_model.generate_greedy_logprobs(prompts,
269
+ max_tokens,
270
+ num_logprobs=num_logprobs,
271
+ images=images or None,
272
+ videos=videos or None)
273
+ for prompts, images, videos in inputs
274
+ ]
275
+
276
+ outputs_per_case_for_embeddings_input = [
277
+ vllm_model.generate_greedy_logprobs(
278
+ prompts,
279
+ max_tokens,
280
+ num_logprobs=num_logprobs,
281
+ images=batch_make_image_embeddings(
282
+ images, processor, vllm_model.model) if images else None,
283
+ videos=batch_make_video_embeddings(
284
+ videos, processor, vllm_model.model) if videos else None)
285
+ for prompts, images, videos in inputs
286
+ ]
287
+
288
+ for outputs_for_original_input, \
289
+ outputs_for_embeddings_input \
290
+ in zip(outputs_per_case_for_original_input,
291
+ outputs_per_case_for_embeddings_input):
292
+ check_logprobs_close(
293
+ outputs_0_lst=outputs_for_original_input,
294
+ outputs_1_lst=outputs_for_embeddings_input,
295
+ name_0="original_input",
296
+ name_1="embeddings_input",
297
+ )
298
+
299
+
300
+@pytest.mark.core_model
301
+@pytest.mark.parametrize("model", models)
302
+@pytest.mark.parametrize(
303
+ "size_factors",
304
+ [
305
+ # Single-scale
306
+ [0.5],
307
+ # Single-scale, batched
308
+ [0.5, 0.5],
309
+ # Multi-scale
310
+ [0.25, 0.5, 0.5],
311
+ ],
312
+)
313
+@pytest.mark.parametrize("dtype", [target_dtype])
314
+@pytest.mark.parametrize("max_tokens", [128])
315
+@pytest.mark.parametrize("num_logprobs", [10])
316
+def test_qwen2_vl_image_embeddings_input(vllm_runner, image_assets, model,
317
+ size_factors, dtype: str,
318
+ max_tokens: int,
319
+ num_logprobs: int) -> None:
320
+ images = [asset.pil_image for asset in image_assets]
321
+
322
+ inputs_per_case: List[Tuple[
323
+ List[str], PromptImageInput, PromptVideoInput]] = [(
324
+ [prompt for _ in size_factors],
325
+ [rescale_image_size(image, factor) for factor in size_factors],
326
+ [],
327
+ ) for image, prompt in zip(images, IMAGE_PROMPTS)]
328
+
329
+ run_test(
330
+ vllm_runner,
331
+ inputs_per_case,
332
+ model,
333
+ dtype=dtype,
334
+ max_tokens=max_tokens,
335
+ num_logprobs=num_logprobs,
336
+ mm_limit=1,
337
+ tensor_parallel_size=1,
338
+ )
339
+
340
+
341
+@pytest.mark.core_model
342
+@pytest.mark.parametrize("model", models)
343
+@pytest.mark.parametrize(
344
+ "size_factors",
345
+ [
346
+ [],
347
+ # Single-scale
348
+ [0.5],
349
+ # Single-scale, batched
350
+ [0.5, 0.5],
351
+ # Multi-scale
352
+ [0.25, 0.5, 0.5],
353
+ ],
354
+)
355
+@pytest.mark.parametrize("dtype", [target_dtype])
356
+@pytest.mark.parametrize("max_tokens", [128])
357
+@pytest.mark.parametrize("num_logprobs", [10])
358
+def test_qwen2_vl_multiple_image_embeddings_input(vllm_runner, image_assets,
359
+ model, size_factors,
360
+ dtype: str, max_tokens: int,
361
+ num_logprobs: int) -> None:
362
+ images = [asset.pil_image for asset in image_assets]
363
+
364
+ inputs_per_case: List[Tuple[List[str], PromptImageInput,
365
+ PromptVideoInput]] = [(
366
+ [MULTIIMAGE_PROMPT for _ in size_factors],
367
+ [[
368
+ rescale_image_size(image, factor)
369
+ for image in images
370
+ ] for factor in size_factors],
371
+ [],
372
+ )]
373
+
374
+ run_test(
375
+ vllm_runner,
376
+ inputs_per_case,
377
+ model,
378
+ dtype=dtype,
379
+ max_tokens=max_tokens,
380
+ num_logprobs=num_logprobs,
381
+ mm_limit=2,
382
+ tensor_parallel_size=1,
383
+ )
384
+
385
+
386
+@pytest.mark.core_model
387
+@pytest.mark.parametrize("model", models)
388
+@pytest.mark.parametrize(
389
+ "size_factors",
390
+ [
391
+ # Single-scale
392
+ [0.5],
393
+ # Single-scale, batched
394
+ [0.5, 0.5],
395
+ # Multi-scale
396
+ [0.25, 0.25, 0.5],
397
+ ],
398
+)
399
+@pytest.mark.parametrize("dtype", [target_dtype])
400
+@pytest.mark.parametrize("max_tokens", [128])
401
+@pytest.mark.parametrize("num_logprobs", [10])
402
+def test_qwen2_vl_video_embeddings_input(vllm_runner, video_assets, model,
403
+ size_factors, dtype: str,
404
+ max_tokens: int,
405
+ num_logprobs: int) -> None:
406
+ num_frames = 4
407
+ sampled_vids = [
408
+ sample_frames_from_video(asset.np_ndarrays, num_frames)
409
+ for asset in video_assets
410
+ ]
411
+
412
+ inputs_per_case: List[Tuple[
413
+ List[str], PromptImageInput, PromptVideoInput]] = [(
414
+ [prompt for _ in size_factors],
415
+ [],
416
+ [rescale_video_size(video, factor) for factor in size_factors],
417
+ ) for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)]
418
+
419
+ run_test(
420
+ vllm_runner,
421
+ inputs_per_case,
422
+ model,
423
+ dtype=dtype,
424
+ max_tokens=max_tokens,
425
+ num_logprobs=num_logprobs,
426
+ mm_limit=1,
427
+ tensor_parallel_size=1,
428
+ )
vllm/model_executor/models/qwen2_vl.py
+149
-31
@@ -79,7 +79,7 @@ logger = init_logger(__name__)
79
80
class Qwen2VLImagePixelInputs(TypedDict):
81
type: Literal["pixel_values"]
82
- data: torch.Tensor
82
+ pixel_values: torch.Tensor
83
"""Shape:
84
`(num_patches, num_channels * patch_size * patch_size)`
85
"""
@@ -92,9 +92,22 @@ class Qwen2VLImagePixelInputs(TypedDict):
92
93
class Qwen2VLImageEmbeddingInputs(TypedDict):
94
type: Literal["image_embeds"]
95
- data: torch.Tensor
96
- """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
97
- `hidden_size` must match the hidden size of language model backbone.
95
+ image_embeds: torch.Tensor
96
+ """Supported types:
97
+ - List[`torch.Tensor`]: A list of tensors holding all images' features.
98
+ Each tensor holds an image's features.
99
+ - `torch.Tensor`: A tensor holding all images' features
100
+ (concatenation of all images' feature tensors).
101
+
102
+ Tensor shape: `(num_image_features, hidden_size)`
103
+ - `num_image_features` varies based on
104
+ the number and resolution of the images.
105
+ - `hidden_size` must match the hidden size of language model backbone.
106
+ """
107
+
108
+ image_grid_thw: torch.Tensor
109
+ """Shape: `(num_images, 3)`
110
+ This should be in `(grid_t, grid_h, grid_w)` format.
111
"""
112
113
@@ -102,7 +115,8 @@ Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
115
Qwen2VLImageEmbeddingInputs]
116
117
105
-class Qwen2VLVideoInputs(TypedDict):
118
+class Qwen2VLVideoPixelInputs(TypedDict):
119
+ type: Literal["pixel_values_videos"]
120
pixel_values_videos: torch.Tensor
121
"""Shape:
122
`(num_patches,
@@ -116,6 +130,30 @@ class Qwen2VLVideoInputs(TypedDict):
130
"""
131
132
133
+class Qwen2VLVideoEmbeddingInputs(TypedDict):
134
+ type: Literal["video_embeds"]
135
+ video_embeds: torch.Tensor
136
+ """Supported types:
137
+ - List[`torch.Tensor`]: A list of tensors holding all videos' features.
138
+ Each tensor holds an video's features.
139
+ - `torch.Tensor`: A tensor holding all videos' features
140
+ (concatenation of all videos' feature tensors).
141
+
142
+ Tensor shape: `(num_image_features, hidden_size)`
143
+ - `num_image_features` varies based on
144
+ the number and resolution of the videos.
145
+ - `hidden_size` must match the hidden size of language model backbone.
146
+ """
147
+
148
+ video_grid_thw: torch.Tensor
149
+ """Shape: `(num_videos, 3)`
150
+ This should be in `(grid_t, grid_h, grid_w)` format.
151
+ """
152
+
153
+
154
+Qwen2VLVideoInputs = Union[Qwen2VLVideoPixelInputs,
155
+ Qwen2VLVideoEmbeddingInputs]
156
+
157
# === Vision Encoder === #
158
159
@@ -585,6 +623,12 @@ def mm_input_mapper_for_qwen2_vl(
623
"image_embeds": data.get("image_embeds"),
624
"image_grid_thw": data.get("image_grid_thw"),
625
})
626
+ if data_type_key == "video" and isinstance(data, dict):
627
+ return MultiModalKwargs({
628
+ "video_embeds": data.get("video_embeds"),
629
+ "video_grid_thw": data.get("video_grid_thw"),
630
+ })
631
+
632
model_config = ctx.model_config
633
# Handle mm processor kwargs; we pass these at creation time
634
# because preprocess() in transformers doesn't expose them
@@ -890,16 +934,33 @@ def input_processor_for_qwen2_vl(
934
idx for idx, token in enumerate(prompt_token_ids)
935
if token == hf_config.image_token_id
936
]
893
- image_cnt = len(image_indices)
894
- embed_dim = image_inputs.get('image_embeds').size(0)
895
- assert embed_dim % image_cnt == 0
896
- num_pad_tokens = embed_dim // image_cnt
937
+
938
+ # ensure all image tokens have grid_thw
939
+ assert \
940
+ len(image_indices) == image_inputs["image_grid_thw"].size(0), \
941
+ "image token num does not match image_grid_thw.shape"
942
+
943
+ image_counter = 0
944
+ pad_token_counter = 0
945
for idx, token in enumerate(prompt_token_ids):
946
if idx in image_indices:
947
+ grid_thw = image_inputs["image_grid_thw"][image_counter]
948
+ grid_t, grid_h, grid_w = grid_thw
949
+ num_pad_tokens = (grid_t * grid_h * grid_w //
950
+ image_processor.merge_size //
951
+ image_processor.merge_size)
952
prompt_token_ids_with_image.extend([token] *
953
num_pad_tokens)
954
+ image_counter += 1
955
+ pad_token_counter += num_pad_tokens
956
else:
957
prompt_token_ids_with_image.append(token)
958
+
959
+ # ensure all embeddings are used
960
+ assert \
961
+ pad_token_counter == image_inputs["image_embeds"].size(0), \
962
+ "image_embeds.shape does not match image_grid_thw"
963
+
964
prompt_token_ids = prompt_token_ids_with_image
965
else:
966
prompt_token_ids = _expand_pad_tokens(image_inputs,
@@ -912,14 +973,49 @@ def input_processor_for_qwen2_vl(
973
max_pixels=max_pixels)
974
975
if video_inputs is not None:
915
- prompt_token_ids = _expand_pad_tokens(video_inputs,
916
- hf_config.video_token_id,
917
- make_batched_videos,
918
- "video",
919
- image_processor,
920
- prompt_token_ids,
921
- min_pixels=min_pixels,
922
- max_pixels=max_pixels)
976
+ if isinstance(video_inputs, dict):
977
+ prompt_token_ids_with_video = []
978
+ video_indices = [
979
+ idx for idx, token in enumerate(prompt_token_ids)
980
+ if token == hf_config.video_token_id
981
+ ]
982
+
983
+ # ensure all video tokens have grid_thw
984
+ assert \
985
+ len(video_indices) == video_inputs["video_grid_thw"].size(0), \
986
+ "video token num does not match video_grid_thw.shape"
987
+
988
+ video_counter = 0
989
+ pad_token_counter = 0
990
+ for idx, token in enumerate(prompt_token_ids):
991
+ if idx in video_indices:
992
+ grid_thw = video_inputs["video_grid_thw"][video_counter]
993
+ grid_t, grid_h, grid_w = grid_thw
994
+ num_pad_tokens = (grid_t * grid_h * grid_w //
995
+ image_processor.merge_size //
996
+ image_processor.merge_size)
997
+ prompt_token_ids_with_video.extend([token] *
998
+ num_pad_tokens)
999
+ video_counter += 1
1000
+ pad_token_counter += num_pad_tokens
1001
+ else:
1002
+ prompt_token_ids_with_video.append(token)
1003
+
1004
+ # ensure all embeddings are used
1005
+ assert \
1006
+ pad_token_counter == video_inputs["video_embeds"].size(0), \
1007
+ "video_embeds.shape does not match video_grid_thw"
1008
+
1009
+ prompt_token_ids = prompt_token_ids_with_video
1010
+ else:
1011
+ prompt_token_ids = _expand_pad_tokens(video_inputs,
1012
+ hf_config.video_token_id,
1013
+ make_batched_videos,
1014
+ "video",
1015
+ image_processor,
1016
+ prompt_token_ids,
1017
+ min_pixels=min_pixels,
1018
+ max_pixels=max_pixels)
1019
1020
prompt = inputs.get("prompt")
1021
if prompt is None:
@@ -1051,49 +1147,71 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1147
f"Got type: {type(pixel_values)}")
1148
1149
return Qwen2VLImagePixelInputs(type="pixel_values",
1054
- data=pixel_values,
1150
+ pixel_values=pixel_values,
1151
image_grid_thw=image_grid_thw)
1152
1153
if image_embeds is not None:
1154
image_embeds = self._validate_and_reshape_mm_tensor(
1155
image_embeds, "image embeds")
1156
+ image_grid_thw = self._validate_and_reshape_mm_tensor(
1157
+ image_grid_thw, "image grid_thw")
1158
1159
if not isinstance(image_embeds, torch.Tensor):
1160
raise ValueError("Incorrect type of image embeddings. "
1161
f"Got type: {type(image_embeds)}")
1162
return Qwen2VLImageEmbeddingInputs(type="image_embeds",
1065
- data=image_embeds)
1163
+ image_embeds=image_embeds,
1164
+ image_grid_thw=image_grid_thw)
1165
1166
def _parse_and_validate_video_input(
1167
self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]:
1168
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
1169
+ video_embeds = kwargs.pop("video_embeds", None)
1170
video_grid_thw = kwargs.pop("video_grid_thw", None)
1171
1072
- if pixel_values_videos is None:
1172
+ if pixel_values_videos is None and video_embeds is None:
1173
return None
1174
1075
- pixel_values_videos = self._validate_and_reshape_mm_tensor(
1076
- pixel_values_videos, "video pixel values")
1077
- video_grid_thw = self._validate_and_reshape_mm_tensor(
1078
- video_grid_thw, "video grid_thw")
1079
-
1080
- return Qwen2VLVideoInputs(
1081
- pixel_values_videos=pixel_values_videos,
1082
- video_grid_thw=video_grid_thw,
1083
- )
1175
+ if pixel_values_videos is not None:
1176
+ pixel_values_videos = self._validate_and_reshape_mm_tensor(
1177
+ pixel_values_videos, "video pixel values")
1178
+ video_grid_thw = self._validate_and_reshape_mm_tensor(
1179
+ video_grid_thw, "video grid_thw")
1180
+
1181
+ return Qwen2VLVideoPixelInputs(
1182
+ type="pixel_values_videos",
1183
+ pixel_values_videos=pixel_values_videos,
1184
+ video_grid_thw=video_grid_thw,
1185
+ )
1186
+
1187
+ if video_embeds is not None:
1188
+ video_embeds = self._validate_and_reshape_mm_tensor(
1189
+ video_embeds, "video embeds")
1190
+ video_grid_thw = self._validate_and_reshape_mm_tensor(
1191
+ video_grid_thw, "video grid_thw")
1192
+
1193
+ if not isinstance(video_embeds, torch.Tensor):
1194
+ raise ValueError("Incorrect type of video embeddings. "
1195
+ f"Got type: {type(video_embeds)}")
1196
+ return Qwen2VLVideoEmbeddingInputs(type="video_embeds",
1197
+ video_embeds=video_embeds,
1198
+ video_grid_thw=video_grid_thw)
1199
1200
def _process_image_input(self,
1201
image_input: Qwen2VLImageInputs) -> torch.Tensor:
1202
if image_input["type"] == "image_embeds":
1088
- return image_input["data"].type(self.visual.dtype)
1203
+ return image_input["image_embeds"].type(self.visual.dtype)
1204
1090
- pixel_values = image_input["data"].type(self.visual.dtype)
1205
+ pixel_values = image_input["pixel_values"].type(self.visual.dtype)
1206
image_embeds = self.visual(pixel_values,
1207
grid_thw=image_input["image_grid_thw"])
1208
return image_embeds
1209
1210
def _process_video_input(self,
1211
video_input: Qwen2VLVideoInputs) -> torch.Tensor:
1212
+ if video_input["type"] == "video_embeds":
1213
+ return video_input["video_embeds"].type(self.visual.dtype)
1214
+
1215
pixel_values_videos = video_input["pixel_values_videos"].type(
1216
self.visual.dtype)
1217
video_embeds = self.visual(pixel_values_videos,