@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3945c82346
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model] Add support for Qwen2-VL video embeddings input & multiple image embeddings input with varied resolutions (#10221)

Signed-off-by: imkero <kerorek@outlook.com>

电脑星人 committed Nov 13, 2024 at 15:07 UTC 3945c82346dae3129213607663bfd17edd905fef
3 files changed +578 -32
docs/source/models/supported_models.rst
+1 -1
@@ -538,7 +538,7 @@ Text Generation
538 - ✅︎
539 * - :code:`Qwen2VLForConditionalGeneration`
540 - Qwen2-VL
541 - - T + I\ :sup:`E+` + V\ :sup:`+`
541 + - T + I\ :sup:`E+` + V\ :sup:`E+`
542 - :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
543 - ✅︎
544 - ✅︎
tests/models/decoder_only/vision_language/test_qwen2_vl.py new
+428
@@ -0,0 +1,428 @@
1 +from typing import Any, List, Optional, Tuple, Type, TypedDict, Union
2 +
3 +import numpy.typing as npt
4 +import pytest
5 +import torch
6 +from PIL import Image
7 +
8 +from vllm.entrypoints.llm import LLM
9 +from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
10 + sample_frames_from_video)
11 +
12 +from ....conftest import (IMAGE_ASSETS, VIDEO_ASSETS, PromptImageInput,
13 + PromptVideoInput, VllmRunner)
14 +from ...utils import check_logprobs_close
15 +
16 +models = ["Qwen/Qwen2-VL-2B-Instruct"]
17 +target_dtype = "half"
18 +
19 +IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
20 +VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
21 +
22 +
23 +def qwen2_vl_chat_template(*query):
24 + return f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{''.join(query)}<|im_end|><|im_start|>assistant\n" # noqa: E501
25 +
26 +
27 +IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
28 + "stop_sign":
29 + qwen2_vl_chat_template(
30 + IMAGE_PLACEHOLDER,
31 + "What is the biggest text's content in this image?",
32 + ),
33 + "cherry_blossom":
34 + qwen2_vl_chat_template(
35 + IMAGE_PLACEHOLDER,
36 + "What is the season shown in this image? ",
37 + "Reply with a short sentence (no more than 20 words)",
38 + ),
39 +})
40 +
41 +VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
42 + "sample_demo_1":
43 + qwen2_vl_chat_template(
44 + VIDEO_PLACEHOLDER,
45 + "Describe this video with a short sentence ",
46 + "(no more than 20 words)",
47 + ),
48 +})
49 +
50 +MULTIIMAGE_PROMPT = qwen2_vl_chat_template(
51 + IMAGE_PLACEHOLDER,
52 + IMAGE_PLACEHOLDER,
53 + "Describe these two images separately. ",
54 + "For each image, reply with a short sentence ",
55 + "(no more than 10 words).",
56 +)
57 +
58 +
59 +class Qwen2VLPromptImageEmbeddingInput(TypedDict):
60 + image_embeds: torch.Tensor
61 + image_grid_thw: torch.Tensor
62 +
63 +
64 +class Qwen2VLPromptVideoEmbeddingInput(TypedDict):
65 + video_embeds: torch.Tensor
66 + video_grid_thw: torch.Tensor
67 +
68 +
69 +def batch_make_image_embeddings(
70 + image_batches: List[Union[Image.Image, List[Image.Image]]], processor,
71 + llm: LLM) -> List[Qwen2VLPromptImageEmbeddingInput]:
72 + """batched image embeddings for Qwen2-VL
73 +
74 + This will infer all images' embeddings in a single batch,
75 + and split the result according to input batches.
76 +
77 + image_batches:
78 + - Single-image batches: `List[Image.Image]`
79 + - Multiple-image batches: `List[List[Image.Image]]]`
80 +
81 + returns: `List[Qwen2VLPromptImageEmbeddingInput]`
82 + """
83 +
84 + image_batches_: List[Any] = image_batches[:]
85 +
86 + # convert single-image batches to multiple-image batches
87 + for idx in range(len(image_batches_)):
88 + if not isinstance(image_batches_[idx], list):
89 + image_batches_[idx] = [image_batches_[idx]]
90 +
91 + assert isinstance(image_batches_[idx], list)
92 +
93 + # append all images into a list (as a batch)
94 + images: List[Image.Image] = []
95 + for image_batch in image_batches_:
96 + images += image_batch
97 +
98 + # image to pixel values
99 + image_processor = processor.image_processor
100 +
101 + preprocess_result = image_processor \
102 + .preprocess(images=images, return_tensors="pt") \
103 + .data
104 + pixel_values = preprocess_result["pixel_values"]
105 + image_grid_thw = preprocess_result["image_grid_thw"]
106 +
107 + # pixel values to embeddinds & grid_thws
108 + with torch.no_grad():
109 + visual = llm.llm_engine.model_executor.driver_worker. \
110 + model_runner.model.visual
111 +
112 + pixel_values_on_device = pixel_values.to(visual.device,
113 + dtype=visual.dtype)
114 + image_grid_thw_on_device = image_grid_thw.to(visual.device,
115 + dtype=torch.int64)
116 + image_embeds = visual(pixel_values_on_device,
117 + grid_thw=image_grid_thw_on_device)
118 +
119 + # split into original batches
120 + result: List[Qwen2VLPromptImageEmbeddingInput] = []
121 + image_counter = 0
122 + embed_counter = 0
123 + for image_batch in image_batches_:
124 + cur_batch_image_count = len(image_batch)
125 + merge_size = image_processor.merge_size
126 + cur_batch_embed_len = sum([
127 + grid_thw.prod() // merge_size // merge_size
128 + for grid_thw in image_grid_thw[image_counter:image_counter +
129 + cur_batch_image_count]
130 + ])
131 +
132 + result.append({
133 + "image_embeds":
134 + image_embeds[embed_counter:embed_counter + cur_batch_embed_len],
135 + "image_grid_thw":
136 + image_grid_thw[image_counter:image_counter +
137 + cur_batch_image_count],
138 + })
139 +
140 + embed_counter += cur_batch_embed_len
141 + image_counter += cur_batch_image_count
142 +
143 + # ensure we don't lost any images or embeddings
144 + assert embed_counter == image_embeds.size(0)
145 + assert image_counter == image_grid_thw.size(0)
146 + assert len(image_batches) == len(result)
147 +
148 + return result
149 +
150 +
151 +def batch_make_video_embeddings(
152 + video_batches: PromptVideoInput, processor,
153 + llm: LLM) -> List[Qwen2VLPromptVideoEmbeddingInput]:
154 + """batched video embeddings for Qwen2-VL
155 +
156 + A NDArray represents a single video's all frames.
157 +
158 + This will infer all videos' embeddings in a single batch,
159 + and split the result according to input batches.
160 +
161 + video_batches:
162 + - Single-video batches: `List[NDArray]`
163 + - Multiple-video batches: `List[List[NDArray]]`
164 + """
165 +
166 + video_batches_: List[Any] = video_batches[:]
167 +
168 + for idx in range(len(video_batches_)):
169 + if not isinstance(video_batches_[idx], list):
170 + single_video_batch: List[npt.NDArray] = [video_batches_[idx]]
171 + video_batches_[idx] = single_video_batch
172 +
173 + assert isinstance(video_batches_[idx], list)
174 +
175 + # append all videos into a list (as a batch)
176 + videos: List[npt.NDArray] = []
177 + for video_batch in video_batches_:
178 + videos += video_batch
179 +
180 + # video to pixel values
181 + image_processor = processor.image_processor
182 +
183 + preprocess_result = image_processor \
184 + .preprocess(images=None, videos=videos, return_tensors="pt") \
185 + .data
186 + pixel_values = preprocess_result["pixel_values_videos"]
187 + video_grid_thw = preprocess_result["video_grid_thw"]
188 +
189 + # pixel values to embeddinds & grid_thws
190 + with torch.no_grad():
191 + visual = llm.llm_engine.model_executor.driver_worker.\
192 + model_runner.model.visual
193 +
194 + pixel_values_on_device = pixel_values.to(visual.device,
195 + dtype=visual.dtype)
196 + video_grid_thw_on_device = video_grid_thw.to(visual.device,
197 + dtype=torch.int64)
198 + video_embeds = visual(pixel_values_on_device,
199 + grid_thw=video_grid_thw_on_device)
200 +
201 + # split into original batches
202 + result: List[Qwen2VLPromptVideoEmbeddingInput] = []
203 + video_counter = 0
204 + embed_counter = 0
205 + for video_batch in video_batches_:
206 + cur_batch_video_count = len(video_batch)
207 + merge_size = image_processor.merge_size
208 + cur_batch_embed_len = sum([
209 + grid_thw.prod() // merge_size // merge_size
210 + for grid_thw in video_grid_thw[video_counter:video_counter +
211 + cur_batch_video_count]
212 + ])
213 +
214 + result.append({
215 + "video_embeds":
216 + video_embeds[embed_counter:embed_counter + cur_batch_embed_len],
217 + "video_grid_thw":
218 + video_grid_thw[video_counter:video_counter +
219 + cur_batch_video_count],
220 + })
221 +
222 + embed_counter += cur_batch_embed_len
223 + video_counter += cur_batch_video_count
224 +
225 + # ensure we don't lost any videos or embeddings
226 + assert embed_counter == video_embeds.size(0)
227 + assert video_counter == video_grid_thw.size(0)
228 + assert len(video_batches) == len(result)
229 +
230 + return result
231 +
232 +
233 +def run_test(
234 + vllm_runner: Type[VllmRunner],
235 + inputs: List[Tuple[List[str], PromptImageInput, PromptVideoInput]],
236 + model: str,
237 + *,
238 + dtype: str,
239 + max_tokens: int,
240 + num_logprobs: int,
241 + mm_limit: int,
242 + tensor_parallel_size: int,
243 + distributed_executor_backend: Optional[str] = None,
244 +):
245 + """Inference result should be the same between
246 + original image/video input and image/video embeddings input.
247 + """
248 + from transformers import AutoProcessor # noqa: F401
249 +
250 + processor = AutoProcessor.from_pretrained(model)
251 +
252 + # NOTE:
253 + # max_model_len should be greater than image_feature_size
254 + with vllm_runner(model,
255 + task="generate",
256 + max_model_len=4000,
257 + max_num_seqs=3,
258 + dtype=dtype,
259 + limit_mm_per_prompt={
260 + "image": mm_limit,
261 + "video": mm_limit
262 + },
263 + tensor_parallel_size=tensor_parallel_size,
264 + distributed_executor_backend=distributed_executor_backend
265 + ) as vllm_model:
266 +
267 + outputs_per_case_for_original_input = [
268 + vllm_model.generate_greedy_logprobs(prompts,
269 + max_tokens,
270 + num_logprobs=num_logprobs,
271 + images=images or None,
272 + videos=videos or None)
273 + for prompts, images, videos in inputs
274 + ]
275 +
276 + outputs_per_case_for_embeddings_input = [
277 + vllm_model.generate_greedy_logprobs(
278 + prompts,
279 + max_tokens,
280 + num_logprobs=num_logprobs,
281 + images=batch_make_image_embeddings(
282 + images, processor, vllm_model.model) if images else None,
283 + videos=batch_make_video_embeddings(
284 + videos, processor, vllm_model.model) if videos else None)
285 + for prompts, images, videos in inputs
286 + ]
287 +
288 + for outputs_for_original_input, \
289 + outputs_for_embeddings_input \
290 + in zip(outputs_per_case_for_original_input,
291 + outputs_per_case_for_embeddings_input):
292 + check_logprobs_close(
293 + outputs_0_lst=outputs_for_original_input,
294 + outputs_1_lst=outputs_for_embeddings_input,
295 + name_0="original_input",
296 + name_1="embeddings_input",
297 + )
298 +
299 +
300 +@pytest.mark.core_model
301 +@pytest.mark.parametrize("model", models)
302 +@pytest.mark.parametrize(
303 + "size_factors",
304 + [
305 + # Single-scale
306 + [0.5],
307 + # Single-scale, batched
308 + [0.5, 0.5],
309 + # Multi-scale
310 + [0.25, 0.5, 0.5],
311 + ],
312 +)
313 +@pytest.mark.parametrize("dtype", [target_dtype])
314 +@pytest.mark.parametrize("max_tokens", [128])
315 +@pytest.mark.parametrize("num_logprobs", [10])
316 +def test_qwen2_vl_image_embeddings_input(vllm_runner, image_assets, model,
317 + size_factors, dtype: str,
318 + max_tokens: int,
319 + num_logprobs: int) -> None:
320 + images = [asset.pil_image for asset in image_assets]
321 +
322 + inputs_per_case: List[Tuple[
323 + List[str], PromptImageInput, PromptVideoInput]] = [(
324 + [prompt for _ in size_factors],
325 + [rescale_image_size(image, factor) for factor in size_factors],
326 + [],
327 + ) for image, prompt in zip(images, IMAGE_PROMPTS)]
328 +
329 + run_test(
330 + vllm_runner,
331 + inputs_per_case,
332 + model,
333 + dtype=dtype,
334 + max_tokens=max_tokens,
335 + num_logprobs=num_logprobs,
336 + mm_limit=1,
337 + tensor_parallel_size=1,
338 + )
339 +
340 +
341 +@pytest.mark.core_model
342 +@pytest.mark.parametrize("model", models)
343 +@pytest.mark.parametrize(
344 + "size_factors",
345 + [
346 + [],
347 + # Single-scale
348 + [0.5],
349 + # Single-scale, batched
350 + [0.5, 0.5],
351 + # Multi-scale
352 + [0.25, 0.5, 0.5],
353 + ],
354 +)
355 +@pytest.mark.parametrize("dtype", [target_dtype])
356 +@pytest.mark.parametrize("max_tokens", [128])
357 +@pytest.mark.parametrize("num_logprobs", [10])
358 +def test_qwen2_vl_multiple_image_embeddings_input(vllm_runner, image_assets,
359 + model, size_factors,
360 + dtype: str, max_tokens: int,
361 + num_logprobs: int) -> None:
362 + images = [asset.pil_image for asset in image_assets]
363 +
364 + inputs_per_case: List[Tuple[List[str], PromptImageInput,
365 + PromptVideoInput]] = [(
366 + [MULTIIMAGE_PROMPT for _ in size_factors],
367 + [[
368 + rescale_image_size(image, factor)
369 + for image in images
370 + ] for factor in size_factors],
371 + [],
372 + )]
373 +
374 + run_test(
375 + vllm_runner,
376 + inputs_per_case,
377 + model,
378 + dtype=dtype,
379 + max_tokens=max_tokens,
380 + num_logprobs=num_logprobs,
381 + mm_limit=2,
382 + tensor_parallel_size=1,
383 + )
384 +
385 +
386 +@pytest.mark.core_model
387 +@pytest.mark.parametrize("model", models)
388 +@pytest.mark.parametrize(
389 + "size_factors",
390 + [
391 + # Single-scale
392 + [0.5],
393 + # Single-scale, batched
394 + [0.5, 0.5],
395 + # Multi-scale
396 + [0.25, 0.25, 0.5],
397 + ],
398 +)
399 +@pytest.mark.parametrize("dtype", [target_dtype])
400 +@pytest.mark.parametrize("max_tokens", [128])
401 +@pytest.mark.parametrize("num_logprobs", [10])
402 +def test_qwen2_vl_video_embeddings_input(vllm_runner, video_assets, model,
403 + size_factors, dtype: str,
404 + max_tokens: int,
405 + num_logprobs: int) -> None:
406 + num_frames = 4
407 + sampled_vids = [
408 + sample_frames_from_video(asset.np_ndarrays, num_frames)
409 + for asset in video_assets
410 + ]
411 +
412 + inputs_per_case: List[Tuple[
413 + List[str], PromptImageInput, PromptVideoInput]] = [(
414 + [prompt for _ in size_factors],
415 + [],
416 + [rescale_video_size(video, factor) for factor in size_factors],
417 + ) for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)]
418 +
419 + run_test(
420 + vllm_runner,
421 + inputs_per_case,
422 + model,
423 + dtype=dtype,
424 + max_tokens=max_tokens,
425 + num_logprobs=num_logprobs,
426 + mm_limit=1,
427 + tensor_parallel_size=1,
428 + )
vllm/model_executor/models/qwen2_vl.py
+149 -31
@@ -79,7 +79,7 @@ logger = init_logger(__name__)
79
80 class Qwen2VLImagePixelInputs(TypedDict):
81 type: Literal["pixel_values"]
82 - data: torch.Tensor
82 + pixel_values: torch.Tensor
83 """Shape:
84 `(num_patches, num_channels * patch_size * patch_size)`
85 """
@@ -92,9 +92,22 @@ class Qwen2VLImagePixelInputs(TypedDict):
92
93 class Qwen2VLImageEmbeddingInputs(TypedDict):
94 type: Literal["image_embeds"]
95 - data: torch.Tensor
96 - """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
97 - `hidden_size` must match the hidden size of language model backbone.
95 + image_embeds: torch.Tensor
96 + """Supported types:
97 + - List[`torch.Tensor`]: A list of tensors holding all images' features.
98 + Each tensor holds an image's features.
99 + - `torch.Tensor`: A tensor holding all images' features
100 + (concatenation of all images' feature tensors).
101 +
102 + Tensor shape: `(num_image_features, hidden_size)`
103 + - `num_image_features` varies based on
104 + the number and resolution of the images.
105 + - `hidden_size` must match the hidden size of language model backbone.
106 + """
107 +
108 + image_grid_thw: torch.Tensor
109 + """Shape: `(num_images, 3)`
110 + This should be in `(grid_t, grid_h, grid_w)` format.
111 """
112
113
@@ -102,7 +115,8 @@ Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
115 Qwen2VLImageEmbeddingInputs]
116
117
105 -class Qwen2VLVideoInputs(TypedDict):
118 +class Qwen2VLVideoPixelInputs(TypedDict):
119 + type: Literal["pixel_values_videos"]
120 pixel_values_videos: torch.Tensor
121 """Shape:
122 `(num_patches,
@@ -116,6 +130,30 @@ class Qwen2VLVideoInputs(TypedDict):
130 """
131
132
133 +class Qwen2VLVideoEmbeddingInputs(TypedDict):
134 + type: Literal["video_embeds"]
135 + video_embeds: torch.Tensor
136 + """Supported types:
137 + - List[`torch.Tensor`]: A list of tensors holding all videos' features.
138 + Each tensor holds an video's features.
139 + - `torch.Tensor`: A tensor holding all videos' features
140 + (concatenation of all videos' feature tensors).
141 +
142 + Tensor shape: `(num_image_features, hidden_size)`
143 + - `num_image_features` varies based on
144 + the number and resolution of the videos.
145 + - `hidden_size` must match the hidden size of language model backbone.
146 + """
147 +
148 + video_grid_thw: torch.Tensor
149 + """Shape: `(num_videos, 3)`
150 + This should be in `(grid_t, grid_h, grid_w)` format.
151 + """
152 +
153 +
154 +Qwen2VLVideoInputs = Union[Qwen2VLVideoPixelInputs,
155 + Qwen2VLVideoEmbeddingInputs]
156 +
157 # === Vision Encoder === #
158
159
@@ -585,6 +623,12 @@ def mm_input_mapper_for_qwen2_vl(
623 "image_embeds": data.get("image_embeds"),
624 "image_grid_thw": data.get("image_grid_thw"),
625 })
626 + if data_type_key == "video" and isinstance(data, dict):
627 + return MultiModalKwargs({
628 + "video_embeds": data.get("video_embeds"),
629 + "video_grid_thw": data.get("video_grid_thw"),
630 + })
631 +
632 model_config = ctx.model_config
633 # Handle mm processor kwargs; we pass these at creation time
634 # because preprocess() in transformers doesn't expose them
@@ -890,16 +934,33 @@ def input_processor_for_qwen2_vl(
934 idx for idx, token in enumerate(prompt_token_ids)
935 if token == hf_config.image_token_id
936 ]
893 - image_cnt = len(image_indices)
894 - embed_dim = image_inputs.get('image_embeds').size(0)
895 - assert embed_dim % image_cnt == 0
896 - num_pad_tokens = embed_dim // image_cnt
937 +
938 + # ensure all image tokens have grid_thw
939 + assert \
940 + len(image_indices) == image_inputs["image_grid_thw"].size(0), \
941 + "image token num does not match image_grid_thw.shape"
942 +
943 + image_counter = 0
944 + pad_token_counter = 0
945 for idx, token in enumerate(prompt_token_ids):
946 if idx in image_indices:
947 + grid_thw = image_inputs["image_grid_thw"][image_counter]
948 + grid_t, grid_h, grid_w = grid_thw
949 + num_pad_tokens = (grid_t * grid_h * grid_w //
950 + image_processor.merge_size //
951 + image_processor.merge_size)
952 prompt_token_ids_with_image.extend([token] *
953 num_pad_tokens)
954 + image_counter += 1
955 + pad_token_counter += num_pad_tokens
956 else:
957 prompt_token_ids_with_image.append(token)
958 +
959 + # ensure all embeddings are used
960 + assert \
961 + pad_token_counter == image_inputs["image_embeds"].size(0), \
962 + "image_embeds.shape does not match image_grid_thw"
963 +
964 prompt_token_ids = prompt_token_ids_with_image
965 else:
966 prompt_token_ids = _expand_pad_tokens(image_inputs,
@@ -912,14 +973,49 @@ def input_processor_for_qwen2_vl(
973 max_pixels=max_pixels)
974
975 if video_inputs is not None:
915 - prompt_token_ids = _expand_pad_tokens(video_inputs,
916 - hf_config.video_token_id,
917 - make_batched_videos,
918 - "video",
919 - image_processor,
920 - prompt_token_ids,
921 - min_pixels=min_pixels,
922 - max_pixels=max_pixels)
976 + if isinstance(video_inputs, dict):
977 + prompt_token_ids_with_video = []
978 + video_indices = [
979 + idx for idx, token in enumerate(prompt_token_ids)
980 + if token == hf_config.video_token_id
981 + ]
982 +
983 + # ensure all video tokens have grid_thw
984 + assert \
985 + len(video_indices) == video_inputs["video_grid_thw"].size(0), \
986 + "video token num does not match video_grid_thw.shape"
987 +
988 + video_counter = 0
989 + pad_token_counter = 0
990 + for idx, token in enumerate(prompt_token_ids):
991 + if idx in video_indices:
992 + grid_thw = video_inputs["video_grid_thw"][video_counter]
993 + grid_t, grid_h, grid_w = grid_thw
994 + num_pad_tokens = (grid_t * grid_h * grid_w //
995 + image_processor.merge_size //
996 + image_processor.merge_size)
997 + prompt_token_ids_with_video.extend([token] *
998 + num_pad_tokens)
999 + video_counter += 1
1000 + pad_token_counter += num_pad_tokens
1001 + else:
1002 + prompt_token_ids_with_video.append(token)
1003 +
1004 + # ensure all embeddings are used
1005 + assert \
1006 + pad_token_counter == video_inputs["video_embeds"].size(0), \
1007 + "video_embeds.shape does not match video_grid_thw"
1008 +
1009 + prompt_token_ids = prompt_token_ids_with_video
1010 + else:
1011 + prompt_token_ids = _expand_pad_tokens(video_inputs,
1012 + hf_config.video_token_id,
1013 + make_batched_videos,
1014 + "video",
1015 + image_processor,
1016 + prompt_token_ids,
1017 + min_pixels=min_pixels,
1018 + max_pixels=max_pixels)
1019
1020 prompt = inputs.get("prompt")
1021 if prompt is None:
@@ -1051,49 +1147,71 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1147 f"Got type: {type(pixel_values)}")
1148
1149 return Qwen2VLImagePixelInputs(type="pixel_values",
1054 - data=pixel_values,
1150 + pixel_values=pixel_values,
1151 image_grid_thw=image_grid_thw)
1152
1153 if image_embeds is not None:
1154 image_embeds = self._validate_and_reshape_mm_tensor(
1155 image_embeds, "image embeds")
1156 + image_grid_thw = self._validate_and_reshape_mm_tensor(
1157 + image_grid_thw, "image grid_thw")
1158
1159 if not isinstance(image_embeds, torch.Tensor):
1160 raise ValueError("Incorrect type of image embeddings. "
1161 f"Got type: {type(image_embeds)}")
1162 return Qwen2VLImageEmbeddingInputs(type="image_embeds",
1065 - data=image_embeds)
1163 + image_embeds=image_embeds,
1164 + image_grid_thw=image_grid_thw)
1165
1166 def _parse_and_validate_video_input(
1167 self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]:
1168 pixel_values_videos = kwargs.pop("pixel_values_videos", None)
1169 + video_embeds = kwargs.pop("video_embeds", None)
1170 video_grid_thw = kwargs.pop("video_grid_thw", None)
1171
1072 - if pixel_values_videos is None:
1172 + if pixel_values_videos is None and video_embeds is None:
1173 return None
1174
1075 - pixel_values_videos = self._validate_and_reshape_mm_tensor(
1076 - pixel_values_videos, "video pixel values")
1077 - video_grid_thw = self._validate_and_reshape_mm_tensor(
1078 - video_grid_thw, "video grid_thw")
1079 -
1080 - return Qwen2VLVideoInputs(
1081 - pixel_values_videos=pixel_values_videos,
1082 - video_grid_thw=video_grid_thw,
1083 - )
1175 + if pixel_values_videos is not None:
1176 + pixel_values_videos = self._validate_and_reshape_mm_tensor(
1177 + pixel_values_videos, "video pixel values")
1178 + video_grid_thw = self._validate_and_reshape_mm_tensor(
1179 + video_grid_thw, "video grid_thw")
1180 +
1181 + return Qwen2VLVideoPixelInputs(
1182 + type="pixel_values_videos",
1183 + pixel_values_videos=pixel_values_videos,
1184 + video_grid_thw=video_grid_thw,
1185 + )
1186 +
1187 + if video_embeds is not None:
1188 + video_embeds = self._validate_and_reshape_mm_tensor(
1189 + video_embeds, "video embeds")
1190 + video_grid_thw = self._validate_and_reshape_mm_tensor(
1191 + video_grid_thw, "video grid_thw")
1192 +
1193 + if not isinstance(video_embeds, torch.Tensor):
1194 + raise ValueError("Incorrect type of video embeddings. "
1195 + f"Got type: {type(video_embeds)}")
1196 + return Qwen2VLVideoEmbeddingInputs(type="video_embeds",
1197 + video_embeds=video_embeds,
1198 + video_grid_thw=video_grid_thw)
1199
1200 def _process_image_input(self,
1201 image_input: Qwen2VLImageInputs) -> torch.Tensor:
1202 if image_input["type"] == "image_embeds":
1088 - return image_input["data"].type(self.visual.dtype)
1203 + return image_input["image_embeds"].type(self.visual.dtype)
1204
1090 - pixel_values = image_input["data"].type(self.visual.dtype)
1205 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)
1206 image_embeds = self.visual(pixel_values,
1207 grid_thw=image_input["image_grid_thw"])
1208 return image_embeds
1209
1210 def _process_video_input(self,
1211 video_input: Qwen2VLVideoInputs) -> torch.Tensor:
1212 + if video_input["type"] == "video_embeds":
1213 + return video_input["video_embeds"].type(self.visual.dtype)
1214 +
1215 pixel_values_videos = video_input["pixel_values_videos"].type(
1216 self.visual.dtype)
1217 video_embeds = self.visual(pixel_values_videos,