@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6202daeed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model] Support multiple images for qwen-vl (#8247)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Alex Brooks committed Sep 12, 2024 at 11:10 UTC c6202daeedb22cd675942c37ae5e194549803c89
4 files changed +343 -65
docs/source/models/supported_models.rst
+1 -1
@@ -254,7 +254,7 @@ Multimodal Language Models
254 -
255 * - :code:`QWenLMHeadModel`
256 - Qwen-VL
257 - - Image\ :sup:`E`
257 + - Image\ :sup:`E+`
258 - :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
259 -
260 * - :code:`Qwen2VLForConditionalGeneration`
examples/offline_inference_vision_language_multi_image.py
+60 -24
@@ -19,7 +19,39 @@ IMAGE_URLS = [
19 ]
20
21
22 -def load_phi3v(question, image_urls: List[str]):
22 +def load_qwenvl_chat(question: str, image_urls: List[str]):
23 + model_name = "Qwen/Qwen-VL-Chat"
24 + llm = LLM(
25 + model=model_name,
26 + trust_remote_code=True,
27 + max_num_seqs=5,
28 + limit_mm_per_prompt={"image": len(image_urls)},
29 + )
30 + placeholders = "".join(f"Picture {i}: <img></img>\n"
31 + for i, _ in enumerate(image_urls, start=1))
32 +
33 + # This model does not have a chat_template attribute on its tokenizer,
34 + # so we need to explicitly pass it. We use ChatML since it's used in the
35 + # generation utils of the model:
36 + # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
37 + tokenizer = AutoTokenizer.from_pretrained(model_name,
38 + trust_remote_code=True)
39 +
40 + # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
41 + chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501
42 +
43 + messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
44 + prompt = tokenizer.apply_chat_template(messages,
45 + tokenize=False,
46 + add_generation_prompt=True,
47 + chat_template=chat_template)
48 +
49 + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
50 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
51 + return llm, prompt, stop_token_ids, None, chat_template
52 +
53 +
54 +def load_phi3v(question: str, image_urls: List[str]):
55 llm = LLM(
56 model="microsoft/Phi-3.5-vision-instruct",
57 trust_remote_code=True,
@@ -30,10 +62,10 @@ def load_phi3v(question, image_urls: List[str]):
62 for i, _ in enumerate(image_urls, start=1))
63 prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
64 stop_token_ids = None
33 - return llm, prompt, stop_token_ids, None
65 + return llm, prompt, stop_token_ids, None, None
66
67
36 -def load_internvl(question, image_urls: List[str]):
68 +def load_internvl(question: str, image_urls: List[str]):
69 model_name = "OpenGVLab/InternVL2-2B"
70
71 llm = LLM(
@@ -61,7 +93,7 @@ def load_internvl(question, image_urls: List[str]):
93 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
94 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
95
64 - return llm, prompt, stop_token_ids, None
96 + return llm, prompt, stop_token_ids, None, None
97
98
99 def load_qwen2_vl(question, image_urls: List[str]):
@@ -111,18 +143,19 @@ def load_qwen2_vl(question, image_urls: List[str]):
143 else:
144 image_data, _ = process_vision_info(messages)
145
114 - return llm, prompt, stop_token_ids, image_data
146 + return llm, prompt, stop_token_ids, image_data, None
147
148
149 model_example_map = {
150 "phi3_v": load_phi3v,
151 "internvl_chat": load_internvl,
152 "qwen2_vl": load_qwen2_vl,
153 + "qwen_vl_chat": load_qwenvl_chat,
154 }
155
156
157 def run_generate(model, question: str, image_urls: List[str]):
125 - llm, prompt, stop_token_ids, image_data = model_example_map[model](
158 + llm, prompt, stop_token_ids, image_data, _ = model_example_map[model](
159 question, image_urls)
160 if image_data is None:
161 image_data = [fetch_image(url) for url in image_urls]
@@ -146,29 +179,32 @@ def run_generate(model, question: str, image_urls: List[str]):
179
180
181 def run_chat(model: str, question: str, image_urls: List[str]):
149 - llm, _, stop_token_ids, _ = model_example_map[model](question, image_urls)
182 + llm, _, stop_token_ids, _, chat_template = model_example_map[model](
183 + question, image_urls)
184
185 sampling_params = SamplingParams(temperature=0.0,
186 max_tokens=128,
187 stop_token_ids=stop_token_ids)
154 -
155 - outputs = llm.chat([{
156 - "role":
157 - "user",
158 - "content": [
159 - {
160 - "type": "text",
161 - "text": question,
162 - },
163 - *({
164 - "type": "image_url",
165 - "image_url": {
166 - "url": image_url
188 + outputs = llm.chat(
189 + [{
190 + "role":
191 + "user",
192 + "content": [
193 + {
194 + "type": "text",
195 + "text": question,
196 },
168 - } for image_url in image_urls),
169 - ],
170 - }],
171 - sampling_params=sampling_params)
197 + *({
198 + "type": "image_url",
199 + "image_url": {
200 + "url": image_url
201 + },
202 + } for image_url in image_urls),
203 + ],
204 + }],
205 + sampling_params=sampling_params,
206 + chat_template=chat_template,
207 + )
208
209 for o in outputs:
210 generated_text = o.outputs[0].text
tests/models/test_qwen.py
+273 -35
@@ -1,11 +1,17 @@
1 import pathlib
2 -from typing import List, Optional, Type
2 +from typing import Dict, List, Optional, Tuple, Type, Union
3
4 import pytest
5 +import torch
6 +from PIL.Image import Image
7
6 -from vllm.multimodal.utils import rescale_image_size
8 +from vllm.config import ModelConfig
9 +from vllm.inputs import InputContext, LLMInputs
10 +from vllm.multimodal.base import MultiModalInputs
11 +from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
12
8 -from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13 +from ..conftest import (IMAGE_ASSETS, HfRunner, ImageAsset, PromptImageInput,
14 + VllmRunner, _ImageAssets)
15 from .utils import check_logprobs_close
16
17 pytestmark = pytest.mark.vlm
@@ -23,19 +29,205 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
29 "Picture 1: <img></img>\nWhat is the season?: ",
30 })
31
32 +HF_MULTIIMAGE_IMAGE_PROMPT = "Picture 1: <img></img>\nPicture 2: <img></img>\nCan you compare these images?\n" # noqa: E501
33 +HF_MULTIIMAGE_IMAGE_PROMPT = "Picture 1: <img></img>\nPicture 2: <img></img>\nDescribe the two images in detail.\n" # noqa: E501
34 +### Multimodal preprocessing tests
35 +SAMPLE_IMAGE = IMAGE_ASSETS[0].pil_image
36 +# These values are specific to Qwen-VL/Chat; we can get these from the model
37 +# config also, but they are hardcoded here to keep the parameterize/fixtures
38 +# easy to read.
39 +IMG_START_ID = 151857
40 +IMG_END_ID = 151858
41 +IMG_PAD_ID = 151859
42 +TOKS_PER_IMG = 256
43 +VIS_ENC_DIM = 4096
44 +IMG_SIZE = 448
45 +
46 +
47 +def build_model_context(model_name: str,
48 + tokenizer_name: Optional[str] = None,
49 + trust_remote_code: bool = False):
50 + """Creates an InputContext for a given model.
51 +
52 + Args:
53 + model_name: Name of the model being considered.
54 + tokenizer_name: Name of the tokenizer being considered.
55 + trust_remote_code: Whether or not to allow loading remote code.
56 +
57 + Returns:
58 + InputContext for the model being considered.
59 + """
60 + if tokenizer_name is None:
61 + tokenizer_name = model_name
62 + model_config = ModelConfig(
63 + model_name,
64 + tokenizer_name,
65 + tokenizer_mode="auto",
66 + trust_remote_code=trust_remote_code,
67 + dtype="float32",
68 + seed=0,
69 + )
70 + return InputContext(model_config)
71 +
72 +
73 +@pytest.fixture()
74 +def input_mapper_for_qwen():
75 + # Lazy import to avoid initializing CUDA during test collection
76 + from vllm.model_executor.models.qwen import input_mapper_for_qwen
77 + return input_mapper_for_qwen
78 +
79 +
80 +@pytest.fixture()
81 +def input_processor_for_qwen():
82 + # Lazy import to avoid initializing CUDA during test collection
83 + from vllm.model_executor.models.qwen import input_processor_for_qwen
84 + return input_processor_for_qwen
85 +
86 +
87 +@pytest.fixture()
88 +def qwen_vl_context() -> InputContext:
89 + """Get an InputContext for Qwen-VL."""
90 + return build_model_context(model_name="Qwen/Qwen-VL",
91 + trust_remote_code=True)
92 +
93 +
94 +# Happy path tests for single/multi-image scenarios for the multimodal
95 +# input processor and mapper, respectively
96 +@pytest.mark.parametrize("num_images", [1, 2])
97 +def test_input_processor_valid_mm_data(input_processor_for_qwen,
98 + qwen_vl_context: InputContext,
99 + num_images: int):
100 + """Happy cases for image inputs to Qwen's multimodal input processor."""
101 + prompt = "".join(
102 + [f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
103 + inputs = LLMInputs(
104 + prompt=prompt,
105 + # When processing multimodal data for a multimodal model, the qwen
106 + # input processor will overwrite the provided prompt_token_ids with
107 + # the image prompts
108 + prompt_token_ids=None,
109 + multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
110 + )
111 + proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
112 + assert isinstance(proc_inputs, dict)
113 +
114 + # Each image should have one start / stop and a fixed context of 256
115 + proc_tokens = proc_inputs["prompt_token_ids"]
116 + assert proc_tokens.count(IMG_START_ID) == num_images
117 + assert proc_tokens.count(IMG_END_ID) == num_images
118 + assert proc_tokens.count(IMG_PAD_ID) == num_images * TOKS_PER_IMG
119 +
120 +
121 +@pytest.mark.parametrize(
122 + "img_data,expected_shape",
123 + [
124 + # single / multi-image
125 + (SAMPLE_IMAGE, (1, 3, IMG_SIZE, IMG_SIZE)),
126 + (2 * [SAMPLE_IMAGE], (2, 3, IMG_SIZE, IMG_SIZE)),
127 + # single / multi-image embeddings
128 + (torch.rand(
129 + (TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
130 + (torch.rand(
131 + (1, TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
132 + (torch.rand(
133 + (2, TOKS_PER_IMG, VIS_ENC_DIM)), (2, TOKS_PER_IMG, VIS_ENC_DIM)),
134 + ])
135 +def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
136 + qwen_vl_context: InputContext,
137 + img_data: Union[torch.Tensor, List[Image],
138 + Image],
139 + expected_shape: List[int]):
140 + """Happy cases for image inputs to Qwen's multimodal input mapper."""
141 + mapped_img_data = input_mapper_for_qwen(qwen_vl_context, img_data)
142 + # Ensure that we get the appropriately shaped pixel_values
143 + # for images and image embeddings, respectively.
144 + assert isinstance(mapped_img_data, MultiModalInputs)
145 + assert "pixel_values" in mapped_img_data
146 + assert mapped_img_data["pixel_values"].shape == expected_shape
147 +
148 +
149 +# Sad path tests for the multimodal input processor and mapper, respectively
150 +@pytest.mark.parametrize("mm_data", [
151 + {
152 + "image": torch.rand((5))
153 + },
154 + {
155 + "image": torch.rand((5, 5, 5, 5, 5))
156 + },
157 +])
158 +def test_input_processor_invalid_mm_data(input_processor_for_qwen,
159 + qwen_vl_context: InputContext,
160 + mm_data: Dict[str, torch.Tensor]):
161 + """Test sad cases validated in Qwen's multimodal input processor."""
162 + tokenizer = cached_get_tokenizer(qwen_vl_context.model_config.tokenizer,
163 + trust_remote_code=True)
164 + prompt = "Picture 1: <img></img>\n"
165 + prompt_token_ids = tokenizer.encode(prompt)
166 + inputs = LLMInputs(prompt=prompt,
167 + prompt_token_ids=prompt_token_ids,
168 + multi_modal_data=mm_data)
169 + # Should fail since we have too many or too few dimensions for embeddings
170 + with pytest.raises(ValueError):
171 + input_processor_for_qwen(qwen_vl_context, inputs)
172 +
173 +
174 +@pytest.mark.parametrize(
175 + "img_data",
176 + [
177 + # Wrong context length
178 + torch.rand((1, TOKS_PER_IMG + 10, VIS_ENC_DIM)),
179 + # Wrong visual encoder output size
180 + torch.rand((1, TOKS_PER_IMG, VIS_ENC_DIM + 10)),
181 + ])
182 +def test_input_mapper_invalid_mm_data(
183 + input_mapper_for_qwen,
184 + qwen_vl_context: InputContext,
185 + img_data: Union[torch.Tensor, List[Image], Image],
186 +):
187 + """Sad cases validated in Qwen VL's multimodal input mapper."""
188 + with pytest.raises(ValueError):
189 + input_mapper_for_qwen(qwen_vl_context, img_data)
190 +
191 +
192 +### End-to-end generation tests
193 +def get_prompt_with_path(tmp_path: pathlib.PosixPath, prompt: str,
194 + assets: Union[_ImageAssets, List[ImageAsset]]) -> str:
195 + """Given a temporary dir path, export one or more image assets into the
196 + tempdir & replace its contents with the local path to the string so that
197 + the HF version of Qwen-VL can resolve the path and load the image ni its
198 + forward() call.
199 +
200 + Args:
201 + tmp_path: Tempdir for test under consideration.
202 + prompt: Prompt with image placeholders.
203 + assets: List of image assets whose len equals the num placeholders.
204 + """
205 + # Ensure that the number of placeholders matches the number of assets;
206 + # If this is not true, the test is probably written incorrectly.
207 + assert prompt.count("<img></img>") == len(assets)
208 +
209 + # Replace the placeholders with local paths to the exported assets
210 + for asset in assets:
211 + image_tmp_path = tmp_path / f"{asset.name}.jpg"
212 + asset.pil_image.save(image_tmp_path)
213 + prompt = prompt.replace(
214 + "<img></img>",
215 + f"<img>{image_tmp_path}</img>",
216 + 1,
217 + )
218 + return prompt
219 +
220
27 -### Tests for multimodal Qwen models
221 def run_test(
29 - tmp_path: pathlib.PosixPath,
222 hf_runner: Type[HfRunner],
223 vllm_runner: Type[VllmRunner],
32 - image_assets: _ImageAssets,
224 + inputs: List[Tuple[List[str], PromptImageInput]],
225 model: str,
226 *,
35 - size_factors: List[float],
227 dtype: str,
228 max_tokens: int,
229 num_logprobs: int,
230 + mm_limit: int,
231 tensor_parallel_size: int,
232 distributed_executor_backend: Optional[str] = None,
233 ):
@@ -48,23 +240,6 @@ def run_test(
240 Note, the text input is also adjusted to abide by vllm contract.
241 The text output is sanitized to be able to compare with hf.
242 """
51 - images = [asset.pil_image for asset in image_assets]
52 -
53 - # Export the images to a tempdir and substitute it into the hf prompt;
54 - # the contents between <img>/</img> will be ignored by VLLM, but the
55 - # transformers implementation for the visual transformer parses this to
56 - # reload it in the forward call; the contents are treated as a URL or a
57 - # local path.
58 - for idx, asset in enumerate(image_assets):
59 - image_tmp_path = tmp_path / f"{asset.name}.jpg"
60 - asset.pil_image.save(image_tmp_path)
61 - HF_IMAGE_PROMPTS[idx] = HF_IMAGE_PROMPTS[idx].replace(
62 - "<img></img>", f"<img>{image_tmp_path}</img>")
63 -
64 - inputs_per_image = [(
65 - [prompt for _ in size_factors],
66 - [rescale_image_size(image, factor) for factor in size_factors],
67 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
243
244 # NOTE: take care of the order. run vLLM first, and then run HF.
245 # vLLM needs a fresh new process without cuda initialization.
@@ -72,11 +247,12 @@ def run_test(
247 # will hurt multiprocessing backend with fork method (the default method).
248
249 # max_model_len should be greater than image_feature_size
75 - # Qwen encodes images into a fixed content size of 256
250 + # Qwen encodes each image into a fixed content size of 256
251 with vllm_runner(model,
77 - max_model_len=300,
252 + max_model_len=1024,
253 max_num_seqs=1,
254 dtype=dtype,
255 + limit_mm_per_prompt={"image": mm_limit},
256 tensor_parallel_size=tensor_parallel_size,
257 distributed_executor_backend=distributed_executor_backend,
258 enforce_eager=True) as vllm_model:
@@ -85,7 +261,7 @@ def run_test(
261 max_tokens,
262 num_logprobs=num_logprobs,
263 images=images)
88 - for prompts, images in inputs_per_image
264 + for prompts, images in inputs
265 ]
266
267 with hf_runner(model, dtype=dtype) as hf_model:
@@ -94,7 +270,7 @@ def run_test(
270 max_tokens,
271 num_logprobs=num_logprobs,
272 images=images)
97 - for prompts, images in inputs_per_image
273 + for prompts, images in inputs
274 ]
275
276 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -125,19 +301,81 @@ def run_test(
301 @pytest.mark.parametrize("dtype", ["bfloat16"])
302 @pytest.mark.parametrize("max_tokens", [8])
303 @pytest.mark.parametrize("num_logprobs", [5])
128 -def test_multimodal_models(tmp_path, hf_runner, vllm_runner, image_assets,
129 - model, size_factors, dtype, max_tokens,
130 - num_logprobs) -> None:
304 +def test_multimodal_models_single_image(tmp_path: pathlib.PosixPath,
305 + hf_runner: Type[HfRunner],
306 + vllm_runner: Type[VllmRunner],
307 + image_assets: _ImageAssets, model: str,
308 + size_factors: List[float], dtype: str,
309 + max_tokens: int,
310 + num_logprobs: int) -> None:
311 + """Tests multimodal models with single image prompts."""
312 + images = [asset.pil_image for asset in image_assets]
313 +
314 + prompts = [
315 + get_prompt_with_path(tmp_path, prompt, [asset])
316 + for prompt, asset in zip(HF_IMAGE_PROMPTS, image_assets)
317 + ]
318 +
319 + inputs = [(
320 + [prompt for _ in size_factors],
321 + [rescale_image_size(image, factor) for factor in size_factors],
322 + ) for image, prompt in zip(images, prompts)]
323 +
324 + run_test(
325 + hf_runner,
326 + vllm_runner,
327 + inputs,
328 + model,
329 + dtype=dtype,
330 + max_tokens=max_tokens,
331 + num_logprobs=num_logprobs,
332 + mm_limit=1,
333 + tensor_parallel_size=1,
334 + )
335 +
336 +
337 +@pytest.mark.parametrize("model", multimodal_models)
338 +@pytest.mark.parametrize(
339 + "size_factors",
340 + [
341 + # No image
342 + [],
343 + # Single-scale
344 + [1.0],
345 + # Single-scale, batched
346 + [1.0, 1.0, 1.0],
347 + # Multi-scale
348 + [0.25, 0.5, 1.0],
349 + ],
350 +)
351 +@pytest.mark.parametrize("dtype", ["bfloat16"])
352 +@pytest.mark.parametrize("max_tokens", [128])
353 +@pytest.mark.parametrize("num_logprobs", [5])
354 +def test_multimodal_models_multi_image(tmp_path: pathlib.PosixPath,
355 + hf_runner: Type[HfRunner],
356 + vllm_runner: Type[VllmRunner],
357 + image_assets: _ImageAssets, model: str,
358 + size_factors: List[float], dtype: str,
359 + max_tokens: int,
360 + num_logprobs: int) -> None:
361 + """Tests multimodal models with multi-image prompts."""
362 + images = [asset.pil_image for asset in image_assets]
363 + # Put all of the images into one prompt.
364 + prompt = get_prompt_with_path(tmp_path, HF_MULTIIMAGE_IMAGE_PROMPT,
365 + image_assets)
366 + inputs = [([prompt for _ in size_factors],
367 + [[rescale_image_size(image, factor) for image in images]
368 + for factor in size_factors])]
369 +
370 run_test(
132 - tmp_path,
371 hf_runner,
372 vllm_runner,
135 - image_assets,
373 + inputs,
374 model,
137 - size_factors=size_factors,
375 dtype=dtype,
376 max_tokens=max_tokens,
377 num_logprobs=num_logprobs,
378 + mm_limit=2,
379 tensor_parallel_size=1,
380 )
381
@@ -150,7 +388,7 @@ def test_multimodal_models(tmp_path, hf_runner, vllm_runner, image_assets,
388 @pytest.mark.parametrize("num_logprobs", [5])
389 def test_text_only_qwen_model_can_be_loaded_and_run(
390 vllm_runner: Type[VllmRunner],
153 - example_prompts,
391 + example_prompts: List[str],
392 model: str,
393 *,
394 dtype: str,
vllm/model_executor/models/qwen.py
+9 -5
@@ -47,6 +47,7 @@ from vllm.multimodal.base import MultiModalInputs
47 from vllm.multimodal.utils import cached_get_tokenizer
48 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
49 SequenceData)
50 +from vllm.utils import is_list_of
51
52 from .utils import flatten_bn, is_pp_missing_parameter, make_layers
53
@@ -684,9 +685,12 @@ def input_processor_for_qwen(ctx: InputContext,
685 raise ValueError(
686 f"Expected img embeds to be have 3 dimensions, got {num_dims}")
687 num_images = 1 if num_dims == 2 else image_data.shape[0]
687 - else:
688 - # TODO - handle multiple image inputs once the API is solidified
688 + elif isinstance(image_data, Image.Image):
689 num_images = 1
690 + elif is_list_of(image_data, Image.Image):
691 + num_images = len(image_data)
692 + else:
693 + raise TypeError(f"Invalid image type: {type(image_data)}")
694
695 if prompt is None:
696 prompt = tokenizer.decode(prompt_token_ids)
@@ -767,11 +771,11 @@ def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
771 f"[# images, {MAX_QWEN_IMG_TOKENS}, {img_emb_size}], but "
772 f"received shape [{data.shape}]")
773 pixel_values = data
770 -
774 else:
775 transform = build_normalization_transform(image_size)
773 - # TODO - handle multiple image inputs once the API is solidified
774 - transformed_images = [transform(data)]
776 + if not isinstance(data, (list, tuple)):
777 + data = [data]
778 + transformed_images = [transform(datum) for datum in data]
779 pixel_values = torch.stack(transformed_images, dim=0)
780 return MultiModalInputs({"pixel_values": pixel_values})
781