A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model] Support multiple images for qwen-vl (#8247)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Alex Brooks committed
Sep 12, 2024 at 11:10 UTC
c6202daeedb22cd675942c37ae5e194549803c89
4 files changed
+343
-65
docs/source/models/supported_models.rst
+1
-1
@@ -254,7 +254,7 @@ Multimodal Language Models
254
-
255
* - :code:`QWenLMHeadModel`
256
- Qwen-VL
257
- - Image\ :sup:`E`
257
+ - Image\ :sup:`E+`
258
- :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
259
-
260
* - :code:`Qwen2VLForConditionalGeneration`
examples/offline_inference_vision_language_multi_image.py
+60
-24
@@ -19,7 +19,39 @@ IMAGE_URLS = [
19
]
20
21
22
-def load_phi3v(question, image_urls: List[str]):
22
+def load_qwenvl_chat(question: str, image_urls: List[str]):
23
+ model_name = "Qwen/Qwen-VL-Chat"
24
+ llm = LLM(
25
+ model=model_name,
26
+ trust_remote_code=True,
27
+ max_num_seqs=5,
28
+ limit_mm_per_prompt={"image": len(image_urls)},
29
+ )
30
+ placeholders = "".join(f"Picture {i}: <img></img>\n"
31
+ for i, _ in enumerate(image_urls, start=1))
32
+
33
+ # This model does not have a chat_template attribute on its tokenizer,
34
+ # so we need to explicitly pass it. We use ChatML since it's used in the
35
+ # generation utils of the model:
36
+ # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
37
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
38
+ trust_remote_code=True)
39
+
40
+ # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
41
+ chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501
42
+
43
+ messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
44
+ prompt = tokenizer.apply_chat_template(messages,
45
+ tokenize=False,
46
+ add_generation_prompt=True,
47
+ chat_template=chat_template)
48
+
49
+ stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
50
+ stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
51
+ return llm, prompt, stop_token_ids, None, chat_template
52
+
53
+
54
+def load_phi3v(question: str, image_urls: List[str]):
55
llm = LLM(
56
model="microsoft/Phi-3.5-vision-instruct",
57
trust_remote_code=True,
@@ -30,10 +62,10 @@ def load_phi3v(question, image_urls: List[str]):
62
for i, _ in enumerate(image_urls, start=1))
63
prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
64
stop_token_ids = None
33
- return llm, prompt, stop_token_ids, None
65
+ return llm, prompt, stop_token_ids, None, None
66
67
36
-def load_internvl(question, image_urls: List[str]):
68
+def load_internvl(question: str, image_urls: List[str]):
69
model_name = "OpenGVLab/InternVL2-2B"
70
71
llm = LLM(
@@ -61,7 +93,7 @@ def load_internvl(question, image_urls: List[str]):
93
stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
94
stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
95
64
- return llm, prompt, stop_token_ids, None
96
+ return llm, prompt, stop_token_ids, None, None
97
98
99
def load_qwen2_vl(question, image_urls: List[str]):
@@ -111,18 +143,19 @@ def load_qwen2_vl(question, image_urls: List[str]):
143
else:
144
image_data, _ = process_vision_info(messages)
145
114
- return llm, prompt, stop_token_ids, image_data
146
+ return llm, prompt, stop_token_ids, image_data, None
147
148
149
model_example_map = {
150
"phi3_v": load_phi3v,
151
"internvl_chat": load_internvl,
152
"qwen2_vl": load_qwen2_vl,
153
+ "qwen_vl_chat": load_qwenvl_chat,
154
}
155
156
157
def run_generate(model, question: str, image_urls: List[str]):
125
- llm, prompt, stop_token_ids, image_data = model_example_map[model](
158
+ llm, prompt, stop_token_ids, image_data, _ = model_example_map[model](
159
question, image_urls)
160
if image_data is None:
161
image_data = [fetch_image(url) for url in image_urls]
@@ -146,29 +179,32 @@ def run_generate(model, question: str, image_urls: List[str]):
179
180
181
def run_chat(model: str, question: str, image_urls: List[str]):
149
- llm, _, stop_token_ids, _ = model_example_map[model](question, image_urls)
182
+ llm, _, stop_token_ids, _, chat_template = model_example_map[model](
183
+ question, image_urls)
184
185
sampling_params = SamplingParams(temperature=0.0,
186
max_tokens=128,
187
stop_token_ids=stop_token_ids)
154
-
155
- outputs = llm.chat([{
156
- "role":
157
- "user",
158
- "content": [
159
- {
160
- "type": "text",
161
- "text": question,
162
- },
163
- *({
164
- "type": "image_url",
165
- "image_url": {
166
- "url": image_url
188
+ outputs = llm.chat(
189
+ [{
190
+ "role":
191
+ "user",
192
+ "content": [
193
+ {
194
+ "type": "text",
195
+ "text": question,
196
},
168
- } for image_url in image_urls),
169
- ],
170
- }],
171
- sampling_params=sampling_params)
197
+ *({
198
+ "type": "image_url",
199
+ "image_url": {
200
+ "url": image_url
201
+ },
202
+ } for image_url in image_urls),
203
+ ],
204
+ }],
205
+ sampling_params=sampling_params,
206
+ chat_template=chat_template,
207
+ )
208
209
for o in outputs:
210
generated_text = o.outputs[0].text
tests/models/test_qwen.py
+273
-35
@@ -1,11 +1,17 @@
1
import pathlib
2
-from typing import List, Optional, Type
2
+from typing import Dict, List, Optional, Tuple, Type, Union
3
4
import pytest
5
+import torch
6
+from PIL.Image import Image
7
6
-from vllm.multimodal.utils import rescale_image_size
8
+from vllm.config import ModelConfig
9
+from vllm.inputs import InputContext, LLMInputs
10
+from vllm.multimodal.base import MultiModalInputs
11
+from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
12
8
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13
+from ..conftest import (IMAGE_ASSETS, HfRunner, ImageAsset, PromptImageInput,
14
+ VllmRunner, _ImageAssets)
15
from .utils import check_logprobs_close
16
17
pytestmark = pytest.mark.vlm
@@ -23,19 +29,205 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
29
"Picture 1: <img></img>\nWhat is the season?: ",
30
})
31
32
+HF_MULTIIMAGE_IMAGE_PROMPT = "Picture 1: <img></img>\nPicture 2: <img></img>\nCan you compare these images?\n" # noqa: E501
33
+HF_MULTIIMAGE_IMAGE_PROMPT = "Picture 1: <img></img>\nPicture 2: <img></img>\nDescribe the two images in detail.\n" # noqa: E501
34
+### Multimodal preprocessing tests
35
+SAMPLE_IMAGE = IMAGE_ASSETS[0].pil_image
36
+# These values are specific to Qwen-VL/Chat; we can get these from the model
37
+# config also, but they are hardcoded here to keep the parameterize/fixtures
38
+# easy to read.
39
+IMG_START_ID = 151857
40
+IMG_END_ID = 151858
41
+IMG_PAD_ID = 151859
42
+TOKS_PER_IMG = 256
43
+VIS_ENC_DIM = 4096
44
+IMG_SIZE = 448
45
+
46
+
47
+def build_model_context(model_name: str,
48
+ tokenizer_name: Optional[str] = None,
49
+ trust_remote_code: bool = False):
50
+ """Creates an InputContext for a given model.
51
+
52
+ Args:
53
+ model_name: Name of the model being considered.
54
+ tokenizer_name: Name of the tokenizer being considered.
55
+ trust_remote_code: Whether or not to allow loading remote code.
56
+
57
+ Returns:
58
+ InputContext for the model being considered.
59
+ """
60
+ if tokenizer_name is None:
61
+ tokenizer_name = model_name
62
+ model_config = ModelConfig(
63
+ model_name,
64
+ tokenizer_name,
65
+ tokenizer_mode="auto",
66
+ trust_remote_code=trust_remote_code,
67
+ dtype="float32",
68
+ seed=0,
69
+ )
70
+ return InputContext(model_config)
71
+
72
+
73
+@pytest.fixture()
74
+def input_mapper_for_qwen():
75
+ # Lazy import to avoid initializing CUDA during test collection
76
+ from vllm.model_executor.models.qwen import input_mapper_for_qwen
77
+ return input_mapper_for_qwen
78
+
79
+
80
+@pytest.fixture()
81
+def input_processor_for_qwen():
82
+ # Lazy import to avoid initializing CUDA during test collection
83
+ from vllm.model_executor.models.qwen import input_processor_for_qwen
84
+ return input_processor_for_qwen
85
+
86
+
87
+@pytest.fixture()
88
+def qwen_vl_context() -> InputContext:
89
+ """Get an InputContext for Qwen-VL."""
90
+ return build_model_context(model_name="Qwen/Qwen-VL",
91
+ trust_remote_code=True)
92
+
93
+
94
+# Happy path tests for single/multi-image scenarios for the multimodal
95
+# input processor and mapper, respectively
96
+@pytest.mark.parametrize("num_images", [1, 2])
97
+def test_input_processor_valid_mm_data(input_processor_for_qwen,
98
+ qwen_vl_context: InputContext,
99
+ num_images: int):
100
+ """Happy cases for image inputs to Qwen's multimodal input processor."""
101
+ prompt = "".join(
102
+ [f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
103
+ inputs = LLMInputs(
104
+ prompt=prompt,
105
+ # When processing multimodal data for a multimodal model, the qwen
106
+ # input processor will overwrite the provided prompt_token_ids with
107
+ # the image prompts
108
+ prompt_token_ids=None,
109
+ multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
110
+ )
111
+ proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
112
+ assert isinstance(proc_inputs, dict)
113
+
114
+ # Each image should have one start / stop and a fixed context of 256
115
+ proc_tokens = proc_inputs["prompt_token_ids"]
116
+ assert proc_tokens.count(IMG_START_ID) == num_images
117
+ assert proc_tokens.count(IMG_END_ID) == num_images
118
+ assert proc_tokens.count(IMG_PAD_ID) == num_images * TOKS_PER_IMG
119
+
120
+
121
+@pytest.mark.parametrize(
122
+ "img_data,expected_shape",
123
+ [
124
+ # single / multi-image
125
+ (SAMPLE_IMAGE, (1, 3, IMG_SIZE, IMG_SIZE)),
126
+ (2 * [SAMPLE_IMAGE], (2, 3, IMG_SIZE, IMG_SIZE)),
127
+ # single / multi-image embeddings
128
+ (torch.rand(
129
+ (TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
130
+ (torch.rand(
131
+ (1, TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
132
+ (torch.rand(
133
+ (2, TOKS_PER_IMG, VIS_ENC_DIM)), (2, TOKS_PER_IMG, VIS_ENC_DIM)),
134
+ ])
135
+def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
136
+ qwen_vl_context: InputContext,
137
+ img_data: Union[torch.Tensor, List[Image],
138
+ Image],
139
+ expected_shape: List[int]):
140
+ """Happy cases for image inputs to Qwen's multimodal input mapper."""
141
+ mapped_img_data = input_mapper_for_qwen(qwen_vl_context, img_data)
142
+ # Ensure that we get the appropriately shaped pixel_values
143
+ # for images and image embeddings, respectively.
144
+ assert isinstance(mapped_img_data, MultiModalInputs)
145
+ assert "pixel_values" in mapped_img_data
146
+ assert mapped_img_data["pixel_values"].shape == expected_shape
147
+
148
+
149
+# Sad path tests for the multimodal input processor and mapper, respectively
150
+@pytest.mark.parametrize("mm_data", [
151
+ {
152
+ "image": torch.rand((5))
153
+ },
154
+ {
155
+ "image": torch.rand((5, 5, 5, 5, 5))
156
+ },
157
+])
158
+def test_input_processor_invalid_mm_data(input_processor_for_qwen,
159
+ qwen_vl_context: InputContext,
160
+ mm_data: Dict[str, torch.Tensor]):
161
+ """Test sad cases validated in Qwen's multimodal input processor."""
162
+ tokenizer = cached_get_tokenizer(qwen_vl_context.model_config.tokenizer,
163
+ trust_remote_code=True)
164
+ prompt = "Picture 1: <img></img>\n"
165
+ prompt_token_ids = tokenizer.encode(prompt)
166
+ inputs = LLMInputs(prompt=prompt,
167
+ prompt_token_ids=prompt_token_ids,
168
+ multi_modal_data=mm_data)
169
+ # Should fail since we have too many or too few dimensions for embeddings
170
+ with pytest.raises(ValueError):
171
+ input_processor_for_qwen(qwen_vl_context, inputs)
172
+
173
+
174
+@pytest.mark.parametrize(
175
+ "img_data",
176
+ [
177
+ # Wrong context length
178
+ torch.rand((1, TOKS_PER_IMG + 10, VIS_ENC_DIM)),
179
+ # Wrong visual encoder output size
180
+ torch.rand((1, TOKS_PER_IMG, VIS_ENC_DIM + 10)),
181
+ ])
182
+def test_input_mapper_invalid_mm_data(
183
+ input_mapper_for_qwen,
184
+ qwen_vl_context: InputContext,
185
+ img_data: Union[torch.Tensor, List[Image], Image],
186
+):
187
+ """Sad cases validated in Qwen VL's multimodal input mapper."""
188
+ with pytest.raises(ValueError):
189
+ input_mapper_for_qwen(qwen_vl_context, img_data)
190
+
191
+
192
+### End-to-end generation tests
193
+def get_prompt_with_path(tmp_path: pathlib.PosixPath, prompt: str,
194
+ assets: Union[_ImageAssets, List[ImageAsset]]) -> str:
195
+ """Given a temporary dir path, export one or more image assets into the
196
+ tempdir & replace its contents with the local path to the string so that
197
+ the HF version of Qwen-VL can resolve the path and load the image ni its
198
+ forward() call.
199
+
200
+ Args:
201
+ tmp_path: Tempdir for test under consideration.
202
+ prompt: Prompt with image placeholders.
203
+ assets: List of image assets whose len equals the num placeholders.
204
+ """
205
+ # Ensure that the number of placeholders matches the number of assets;
206
+ # If this is not true, the test is probably written incorrectly.
207
+ assert prompt.count("<img></img>") == len(assets)
208
+
209
+ # Replace the placeholders with local paths to the exported assets
210
+ for asset in assets:
211
+ image_tmp_path = tmp_path / f"{asset.name}.jpg"
212
+ asset.pil_image.save(image_tmp_path)
213
+ prompt = prompt.replace(
214
+ "<img></img>",
215
+ f"<img>{image_tmp_path}</img>",
216
+ 1,
217
+ )
218
+ return prompt
219
+
220
27
-### Tests for multimodal Qwen models
221
def run_test(
29
- tmp_path: pathlib.PosixPath,
222
hf_runner: Type[HfRunner],
223
vllm_runner: Type[VllmRunner],
32
- image_assets: _ImageAssets,
224
+ inputs: List[Tuple[List[str], PromptImageInput]],
225
model: str,
226
*,
35
- size_factors: List[float],
227
dtype: str,
228
max_tokens: int,
229
num_logprobs: int,
230
+ mm_limit: int,
231
tensor_parallel_size: int,
232
distributed_executor_backend: Optional[str] = None,
233
):
@@ -48,23 +240,6 @@ def run_test(
240
Note, the text input is also adjusted to abide by vllm contract.
241
The text output is sanitized to be able to compare with hf.
242
"""
51
- images = [asset.pil_image for asset in image_assets]
52
-
53
- # Export the images to a tempdir and substitute it into the hf prompt;
54
- # the contents between <img>/</img> will be ignored by VLLM, but the
55
- # transformers implementation for the visual transformer parses this to
56
- # reload it in the forward call; the contents are treated as a URL or a
57
- # local path.
58
- for idx, asset in enumerate(image_assets):
59
- image_tmp_path = tmp_path / f"{asset.name}.jpg"
60
- asset.pil_image.save(image_tmp_path)
61
- HF_IMAGE_PROMPTS[idx] = HF_IMAGE_PROMPTS[idx].replace(
62
- "<img></img>", f"<img>{image_tmp_path}</img>")
63
-
64
- inputs_per_image = [(
65
- [prompt for _ in size_factors],
66
- [rescale_image_size(image, factor) for factor in size_factors],
67
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
243
244
# NOTE: take care of the order. run vLLM first, and then run HF.
245
# vLLM needs a fresh new process without cuda initialization.
@@ -72,11 +247,12 @@ def run_test(
247
# will hurt multiprocessing backend with fork method (the default method).
248
249
# max_model_len should be greater than image_feature_size
75
- # Qwen encodes images into a fixed content size of 256
250
+ # Qwen encodes each image into a fixed content size of 256
251
with vllm_runner(model,
77
- max_model_len=300,
252
+ max_model_len=1024,
253
max_num_seqs=1,
254
dtype=dtype,
255
+ limit_mm_per_prompt={"image": mm_limit},
256
tensor_parallel_size=tensor_parallel_size,
257
distributed_executor_backend=distributed_executor_backend,
258
enforce_eager=True) as vllm_model:
@@ -85,7 +261,7 @@ def run_test(
261
max_tokens,
262
num_logprobs=num_logprobs,
263
images=images)
88
- for prompts, images in inputs_per_image
264
+ for prompts, images in inputs
265
]
266
267
with hf_runner(model, dtype=dtype) as hf_model:
@@ -94,7 +270,7 @@ def run_test(
270
max_tokens,
271
num_logprobs=num_logprobs,
272
images=images)
97
- for prompts, images in inputs_per_image
273
+ for prompts, images in inputs
274
]
275
276
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -125,19 +301,81 @@ def run_test(
301
@pytest.mark.parametrize("dtype", ["bfloat16"])
302
@pytest.mark.parametrize("max_tokens", [8])
303
@pytest.mark.parametrize("num_logprobs", [5])
128
-def test_multimodal_models(tmp_path, hf_runner, vllm_runner, image_assets,
129
- model, size_factors, dtype, max_tokens,
130
- num_logprobs) -> None:
304
+def test_multimodal_models_single_image(tmp_path: pathlib.PosixPath,
305
+ hf_runner: Type[HfRunner],
306
+ vllm_runner: Type[VllmRunner],
307
+ image_assets: _ImageAssets, model: str,
308
+ size_factors: List[float], dtype: str,
309
+ max_tokens: int,
310
+ num_logprobs: int) -> None:
311
+ """Tests multimodal models with single image prompts."""
312
+ images = [asset.pil_image for asset in image_assets]
313
+
314
+ prompts = [
315
+ get_prompt_with_path(tmp_path, prompt, [asset])
316
+ for prompt, asset in zip(HF_IMAGE_PROMPTS, image_assets)
317
+ ]
318
+
319
+ inputs = [(
320
+ [prompt for _ in size_factors],
321
+ [rescale_image_size(image, factor) for factor in size_factors],
322
+ ) for image, prompt in zip(images, prompts)]
323
+
324
+ run_test(
325
+ hf_runner,
326
+ vllm_runner,
327
+ inputs,
328
+ model,
329
+ dtype=dtype,
330
+ max_tokens=max_tokens,
331
+ num_logprobs=num_logprobs,
332
+ mm_limit=1,
333
+ tensor_parallel_size=1,
334
+ )
335
+
336
+
337
+@pytest.mark.parametrize("model", multimodal_models)
338
+@pytest.mark.parametrize(
339
+ "size_factors",
340
+ [
341
+ # No image
342
+ [],
343
+ # Single-scale
344
+ [1.0],
345
+ # Single-scale, batched
346
+ [1.0, 1.0, 1.0],
347
+ # Multi-scale
348
+ [0.25, 0.5, 1.0],
349
+ ],
350
+)
351
+@pytest.mark.parametrize("dtype", ["bfloat16"])
352
+@pytest.mark.parametrize("max_tokens", [128])
353
+@pytest.mark.parametrize("num_logprobs", [5])
354
+def test_multimodal_models_multi_image(tmp_path: pathlib.PosixPath,
355
+ hf_runner: Type[HfRunner],
356
+ vllm_runner: Type[VllmRunner],
357
+ image_assets: _ImageAssets, model: str,
358
+ size_factors: List[float], dtype: str,
359
+ max_tokens: int,
360
+ num_logprobs: int) -> None:
361
+ """Tests multimodal models with multi-image prompts."""
362
+ images = [asset.pil_image for asset in image_assets]
363
+ # Put all of the images into one prompt.
364
+ prompt = get_prompt_with_path(tmp_path, HF_MULTIIMAGE_IMAGE_PROMPT,
365
+ image_assets)
366
+ inputs = [([prompt for _ in size_factors],
367
+ [[rescale_image_size(image, factor) for image in images]
368
+ for factor in size_factors])]
369
+
370
run_test(
132
- tmp_path,
371
hf_runner,
372
vllm_runner,
135
- image_assets,
373
+ inputs,
374
model,
137
- size_factors=size_factors,
375
dtype=dtype,
376
max_tokens=max_tokens,
377
num_logprobs=num_logprobs,
378
+ mm_limit=2,
379
tensor_parallel_size=1,
380
)
381
@@ -150,7 +388,7 @@ def test_multimodal_models(tmp_path, hf_runner, vllm_runner, image_assets,
388
@pytest.mark.parametrize("num_logprobs", [5])
389
def test_text_only_qwen_model_can_be_loaded_and_run(
390
vllm_runner: Type[VllmRunner],
153
- example_prompts,
391
+ example_prompts: List[str],
392
model: str,
393
*,
394
dtype: str,
vllm/model_executor/models/qwen.py
+9
-5
@@ -47,6 +47,7 @@ from vllm.multimodal.base import MultiModalInputs
47
from vllm.multimodal.utils import cached_get_tokenizer
48
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
49
SequenceData)
50
+from vllm.utils import is_list_of
51
52
from .utils import flatten_bn, is_pp_missing_parameter, make_layers
53
@@ -684,9 +685,12 @@ def input_processor_for_qwen(ctx: InputContext,
685
raise ValueError(
686
f"Expected img embeds to be have 3 dimensions, got {num_dims}")
687
num_images = 1 if num_dims == 2 else image_data.shape[0]
687
- else:
688
- # TODO - handle multiple image inputs once the API is solidified
688
+ elif isinstance(image_data, Image.Image):
689
num_images = 1
690
+ elif is_list_of(image_data, Image.Image):
691
+ num_images = len(image_data)
692
+ else:
693
+ raise TypeError(f"Invalid image type: {type(image_data)}")
694
695
if prompt is None:
696
prompt = tokenizer.decode(prompt_token_ids)
@@ -767,11 +771,11 @@ def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
771
f"[# images, {MAX_QWEN_IMG_TOKENS}, {img_emb_size}], but "
772
f"received shape [{data.shape}]")
773
pixel_values = data
770
-
774
else:
775
transform = build_normalization_transform(image_size)
773
- # TODO - handle multiple image inputs once the API is solidified
774
- transformed_images = [transform(data)]
776
+ if not isinstance(data, (list, tuple)):
777
+ data = [data]
778
+ transformed_images = [transform(datum) for datum in data]
779
pixel_values = torch.stack(transformed_images, dim=0)
780
return MultiModalInputs({"pixel_values": pixel_values})
781