A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Bugfix] Fix profiling dummy data for Pixtral (#18677)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 25, 2025 at 22:05 UTC
57fd13a70729c73dd4abf97251fdafe28df328d0
5 files changed
+153
-170
tests/models/multimodal/processing/test_common.py
+105
-156
@@ -9,15 +9,15 @@ from mistral_common.protocol.instruct.messages import (ImageChunk, TextChunk,
9
UserMessage)
10
from mistral_common.protocol.instruct.request import ChatCompletionRequest
11
from PIL import Image
12
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
12
13
from vllm.config import ModelConfig
14
from vllm.inputs import InputProcessingContext
15
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
16
from vllm.multimodal.inputs import MultiModalInputs
17
from vllm.multimodal.processing import BaseMultiModalProcessor, ProcessingCache
19
-from vllm.transformers_utils.tokenizer import (MistralTokenizer,
20
- cached_tokenizer_from_config)
18
+from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
19
+ cached_tokenizer_from_config,
20
+ encode_tokens)
21
22
from ....multimodal.utils import random_audio, random_image, random_video
23
from ...registry import HF_EXAMPLE_MODELS
@@ -28,7 +28,6 @@ def _test_processing_correctness(
28
hit_rate: float,
29
num_batches: int,
30
simplify_rate: float,
31
- ignore_mm_keys: Optional[set[str]] = None,
31
):
32
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
33
model_info.check_available_online(on_fail="skip")
@@ -99,10 +98,23 @@ def _test_processing_correctness(
98
}
99
100
mm_counts = {k: len(vs) for k, vs in mm_data.items()}
102
- prompt = dummy_inputs.get_dummy_processor_inputs(
103
- model_config.max_model_len,
104
- mm_counts,
105
- ).prompt_text
101
+
102
+ # Mistral chat outputs tokens directly, rather than text prompts
103
+ if isinstance(tokenizer, MistralTokenizer):
104
+ images = mm_data.get("image", [])
105
+ request = ChatCompletionRequest(messages=[
106
+ UserMessage(content=[
107
+ TextChunk(text=""),
108
+ *(ImageChunk(image=image) for image in images),
109
+ ]),
110
+ ])
111
+ res = tokenizer.mistral.encode_chat_completion(request)
112
+ prompt = res.tokens
113
+ else:
114
+ prompt = dummy_inputs.get_dummy_processor_inputs(
115
+ model_config.max_model_len,
116
+ mm_counts,
117
+ ).prompt
118
119
# Drop unnecessary keys and test single -> multi conversion
120
if rng.rand() < simplify_rate:
@@ -112,67 +124,59 @@ def _test_processing_correctness(
124
elif len(mm_data[k]) == 1:
125
mm_data[k] = mm_data[k][0]
126
115
- if isinstance(tokenizer, MistralTokenizer):
116
- _test_processing_correctness_mistral(
117
- model_config,
118
- tokenizer,
119
- prompt,
120
- mm_data,
121
- baseline_processor,
122
- cached_processor,
123
- batch_idx,
124
- ignore_mm_keys=ignore_mm_keys,
125
- )
126
- else:
127
- _test_processing_correctness_hf(
128
- model_config,
129
- tokenizer,
130
- prompt,
131
- mm_data,
132
- baseline_processor,
133
- cached_processor,
134
- batch_idx,
135
- ignore_mm_keys=ignore_mm_keys,
136
- )
137
-
138
-
139
-def _test_processing_correctness_hf(
127
+ _test_processing_correctness_one(
128
+ model_config,
129
+ tokenizer,
130
+ prompt,
131
+ mm_data,
132
+ baseline_processor,
133
+ cached_processor,
134
+ batch_idx,
135
+ )
136
+
137
+
138
+# For some multimodal models, tokenizer will always add bos_token
139
+# at the beginning of prompt by default, causing hf_processor outputs
140
+# incorrect token ids. So we need use `add_special_tokens=False` here
141
+# to leave bos_token to be added by the processor.
142
+_ADD_SPECIAL_TOKENS_OVERRIDES = {
143
+ "mllama": False,
144
+ "ovis": False,
145
+ "ultravox": False,
146
+ "whisper": False,
147
+}
148
+
149
+_IGNORE_MM_KEYS = {
150
+ # In Ultravox, the audio_features can be different depending on padding
151
+ # The slight difference should not be a problem though, since
152
+ # attention_mask lets us ignore the difference.
153
+ "ultravox": {"audio_features"},
154
+}
155
+
156
+
157
+def _test_processing_correctness_one(
158
model_config: ModelConfig,
141
- tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
142
- prompt: str,
159
+ tokenizer: AnyTokenizer,
160
+ prompt: Union[str, list[int]],
161
mm_data: MultiModalDataDict,
162
baseline_processor: BaseMultiModalProcessor,
163
cached_processor: BaseMultiModalProcessor,
164
batch_idx: int,
147
- ignore_mm_keys: Optional[set[str]] = None,
165
):
149
- if model_config.hf_config.model_type in ("mllama", "ovis", "ultravox",
150
- "whisper"):
151
- # For some multimodal models, tokenizer will always add bos_token
152
- # at the beginning of prompt by default, causing hf_processor outputs
153
- # incorrect token ids. So we need use `add_special_tokens=False` here
154
- # to leave bos_token to be added by the processor.
155
- token_prompt = tokenizer.encode(prompt, add_special_tokens=False)
166
+ model_type = model_config.hf_config.model_type
167
+ ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
168
+
169
+ if isinstance(prompt, str):
170
+ text_prompt = prompt
171
+ token_prompt = encode_tokens(
172
+ tokenizer,
173
+ prompt,
174
+ add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type),
175
+ )
176
else:
157
- token_prompt = tokenizer.encode(prompt)
158
-
159
- baseline_result = baseline_processor.apply(
160
- prompt,
161
- mm_data=mm_data,
162
- hf_processor_mm_kwargs={},
163
- )
164
- cached_result = cached_processor.apply(
165
- prompt,
166
- mm_data=mm_data,
167
- hf_processor_mm_kwargs={},
168
- )
169
-
170
- _assert_inputs_equal(
171
- baseline_result,
172
- cached_result,
173
- ignore_mm_keys=ignore_mm_keys,
174
- msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
175
- )
177
+ # Mistral does not support decode_tokens with skip_special_tokens=False
178
+ text_prompt = None
179
+ token_prompt = prompt
180
181
baseline_tokenized_result = baseline_processor.apply(
182
token_prompt,
@@ -180,56 +184,6 @@ def _test_processing_correctness_hf(
184
hf_processor_mm_kwargs={},
185
)
186
183
- _assert_inputs_equal(
184
- baseline_result,
185
- baseline_tokenized_result,
186
- ignore_mm_keys=ignore_mm_keys,
187
- msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
188
- )
189
-
190
- cached_tokenized_result = cached_processor.apply(
191
- token_prompt,
192
- mm_data=mm_data,
193
- hf_processor_mm_kwargs={},
194
- )
195
-
196
- _assert_inputs_equal(
197
- cached_result,
198
- cached_tokenized_result,
199
- ignore_mm_keys=ignore_mm_keys,
200
- msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
201
- )
202
-
203
-
204
-def _test_processing_correctness_mistral(
205
- model_config: ModelConfig,
206
- tokenizer: MistralTokenizer,
207
- prompt: str,
208
- mm_data: MultiModalDataDict,
209
- baseline_processor: BaseMultiModalProcessor,
210
- cached_processor: BaseMultiModalProcessor,
211
- batch_idx: int,
212
- ignore_mm_keys: Optional[set[str]] = None,
213
-):
214
- images = mm_data.get("image", [])
215
- if not isinstance(images, list):
216
- images = [images]
217
-
218
- request = ChatCompletionRequest(messages=[
219
- UserMessage(content=[
220
- TextChunk(text=prompt),
221
- *(ImageChunk(image=image) for image in images),
222
- ]),
223
- ])
224
- res = tokenizer.mistral.encode_chat_completion(request)
225
- token_prompt = res.tokens
226
-
227
- # Mistral chat outputs tokens directly, rather than text prompts
228
- baseline_tokenized_result = baseline_processor.apply(
229
- token_prompt,
230
- mm_data=mm_data,
231
- hf_processor_mm_kwargs={},
232
- )
187
cached_tokenized_result = cached_processor.apply(
188
token_prompt,
189
mm_data=mm_data,
@@ -240,9 +194,44 @@ def _test_processing_correctness_mistral(
194
baseline_tokenized_result,
195
cached_tokenized_result,
196
ignore_mm_keys=ignore_mm_keys,
243
- msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
197
+ msg=f"Failed ({batch_idx=}, {token_prompt=}, {mm_data=})",
198
)
199
200
+ if text_prompt is not None:
201
+ baseline_text_result = baseline_processor.apply(
202
+ text_prompt,
203
+ mm_data=mm_data,
204
+ hf_processor_mm_kwargs={},
205
+ )
206
+ cached_text_result = cached_processor.apply(
207
+ text_prompt,
208
+ mm_data=mm_data,
209
+ hf_processor_mm_kwargs={},
210
+ )
211
+
212
+ _assert_inputs_equal(
213
+ baseline_text_result,
214
+ cached_text_result,
215
+ ignore_mm_keys=ignore_mm_keys,
216
+ msg=f"Failed ({batch_idx=}, {text_prompt=}, {mm_data=})",
217
+ )
218
+
219
+ _assert_inputs_equal(
220
+ baseline_text_result,
221
+ baseline_tokenized_result,
222
+ ignore_mm_keys=ignore_mm_keys,
223
+ msg=f"Failed ({batch_idx=}, {text_prompt=}, "
224
+ f"{token_prompt=}, {mm_data=})",
225
+ )
226
+
227
+ _assert_inputs_equal(
228
+ cached_text_result,
229
+ cached_tokenized_result,
230
+ ignore_mm_keys=ignore_mm_keys,
231
+ msg=f"Failed ({batch_idx=}, {text_prompt=}, "
232
+ f"{token_prompt=}, {mm_data=})",
233
+ )
234
+
235
236
# yapf: disable
237
@pytest.mark.parametrize("model_id", [
@@ -281,6 +270,7 @@ def _test_processing_correctness_mistral(
270
"AIDC-AI/Ovis2-1B",
271
"google/paligemma-3b-mix-224",
272
"google/paligemma2-3b-ft-docci-448",
273
+ "microsoft/Phi-3.5-vision-instruct",
274
"microsoft/Phi-4-multimodal-instruct",
275
"mistralai/Pixtral-12B-2409",
276
"mistral-community/pixtral-12b",
@@ -303,41 +293,6 @@ def test_processing_correctness(
293
num_batches: int,
294
simplify_rate: float,
295
):
306
- ignore_mm_keys = None
307
- if 'ultravox' in model_id:
308
- # In Ultravox, the audio_features can be different depending on padding
309
- # The slight difference should not be a problem though, since
310
- # attention_mask lets us ignore the difference.
311
- ignore_mm_keys = {"audio_features"}
312
-
313
- _test_processing_correctness(
314
- model_id,
315
- hit_rate=hit_rate,
316
- num_batches=num_batches,
317
- simplify_rate=simplify_rate,
318
- ignore_mm_keys=ignore_mm_keys,
319
- )
320
-
321
-
322
-# yapf: disable
323
-@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
324
-@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
325
-@pytest.mark.parametrize("num_batches", [32])
326
-@pytest.mark.parametrize("simplify_rate", [1.0])
327
-# yapf: enable
328
-def test_processing_correctness_phi3v(
329
- model_id: str,
330
- hit_rate: float,
331
- num_batches: int,
332
- simplify_rate: float,
333
-):
334
- # HACK - this is an attempted workaround for the following bug
335
- # https://github.com/huggingface/transformers/issues/34307
336
- from transformers import AutoImageProcessor # noqa: F401
337
- from transformers import AutoProcessor # noqa: F401
338
-
339
- AutoImageProcessor.from_pretrained(model_id, trust_remote_code=True)
340
-
296
_test_processing_correctness(
297
model_id,
298
hit_rate=hit_rate,
@@ -356,16 +311,10 @@ def _assert_inputs_equal(
311
if ignore_mm_keys is None:
312
ignore_mm_keys = set()
313
359
- if msg is None:
360
- assert "mm_kwargs" in a and "mm_kwargs" in b
361
- else:
362
- assert "mm_kwargs" in a and "mm_kwargs" in b, msg
314
+ assert "mm_kwargs" in a and "mm_kwargs" in b, msg
315
316
for key in ignore_mm_keys:
317
a["mm_kwargs"].pop(key, None)
318
b["mm_kwargs"].pop(key, None)
319
368
- if msg is None:
369
- assert a == b
370
- else:
371
- assert a == b, msg
320
+ assert a == b, msg
tests/models/multimodal/processing/test_mllama.py
+1
-1
@@ -49,7 +49,7 @@ def test_profiling(
49
] * max_num_seqs
50
51
mm_kwargs = processor.apply(
52
- prompt=dummy_mm_data.prompt_text,
52
+ prompt=dummy_mm_data.prompt,
53
mm_data=dummy_mm_data.mm_data,
54
hf_processor_mm_kwargs=dict(),
55
)["mm_kwargs"]
tests/models/registry.py
+5
-4
@@ -8,6 +8,8 @@ import pytest
8
from packaging.version import Version
9
from transformers import __version__ as TRANSFORMERS_VERSION
10
11
+from vllm.config import TokenizerMode
12
+
13
14
@dataclass(frozen=True)
15
class _HfExamplesInfo:
@@ -20,7 +22,7 @@ class _HfExamplesInfo:
22
tokenizer: Optional[str] = None
23
"""Set the tokenizer to load for this architecture."""
24
23
- tokenizer_mode: str = "auto"
25
+ tokenizer_mode: TokenizerMode = "auto"
26
"""Set the tokenizer type for this architecture."""
27
28
speculative_model: Optional[str] = None
@@ -388,8 +390,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
390
"Phi4MMForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct",
391
trust_remote_code=True),
392
"PixtralForConditionalGeneration": _HfExamplesInfo("mistralai/Pixtral-12B-2409", # noqa: E501
391
- tokenizer_mode="mistral",
392
- v0_only=True),
393
+ tokenizer_mode="mistral"),
394
"QwenVLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen-VL",
395
extras={"chat": "Qwen/Qwen-VL-Chat"}, # noqa: E501
396
trust_remote_code=True,
@@ -400,7 +401,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
401
"Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B",
402
min_transformers_version="4.52"),
403
"Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ", # noqa: E501
403
- min_transformers_version="4.52"),
404
+ min_transformers_version="4.52"),
405
"SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),
406
"SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct"), # noqa: E501
407
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
vllm/model_executor/models/pixtral.py
+32
-4
@@ -9,7 +9,9 @@ from typing import Literal, Optional, TypedDict, Union
9
import torch
10
import torch.nn as nn
11
import torch.nn.functional as F
12
-from mistral_common.protocol.instruct.messages import ImageChunk
12
+from mistral_common.protocol.instruct.messages import (ImageChunk, TextChunk,
13
+ UserMessage)
14
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
15
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
16
from PIL import Image
17
from transformers import PixtralVisionConfig, TensorType
@@ -39,7 +41,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
41
BaseProcessingInfo, MultiModalHashes,
42
PromptReplacement, PromptUpdate,
43
PromptUpdateDetails)
42
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
44
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
45
from vllm.sequence import IntermediateTensors
46
from vllm.transformers_utils.tokenizer import (MistralTokenizer,
47
cached_tokenizer_from_config)
@@ -224,6 +226,28 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
226
num_images=num_images)
227
}
228
229
+ def get_dummy_processor_inputs(
230
+ self,
231
+ seq_len: int,
232
+ mm_counts: Mapping[str, int],
233
+ ) -> ProcessorInputs:
234
+ tokenizer = self.info.get_tokenizer()
235
+
236
+ dummy_text = self.get_dummy_text(mm_counts)
237
+ dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)
238
+ dummy_images = dummy_mm_data.get("image", [])
239
+
240
+ request = ChatCompletionRequest(messages=[
241
+ UserMessage(content=[
242
+ TextChunk(text=dummy_text),
243
+ *(ImageChunk(image=image) for image in dummy_images),
244
+ ]),
245
+ ])
246
+ res = tokenizer.mistral.encode_chat_completion(request)
247
+ dummy_tokens = res.tokens
248
+
249
+ return ProcessorInputs(prompt=dummy_tokens, mm_data=dummy_mm_data)
250
+
251
252
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
253
):
@@ -275,8 +299,12 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
299
*,
300
return_mm_hashes: bool,
301
) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
278
- prompt_ids, mm_kwargs, mm_hashes, _ = super(
279
- )._cached_apply_hf_processor(
302
+ (
303
+ prompt_ids,
304
+ mm_kwargs,
305
+ mm_hashes,
306
+ _,
307
+ ) = super()._cached_apply_hf_processor(
308
prompt=prompt,
309
mm_data_items=mm_data_items,
310
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
vllm/multimodal/profiling.py
+10
-5
@@ -3,7 +3,7 @@
3
from abc import ABC
4
from collections.abc import Mapping
5
from dataclasses import dataclass, field
6
-from typing import Generic, NamedTuple, Optional, TypeVar, cast
6
+from typing import Generic, NamedTuple, Optional, TypeVar, Union, cast
7
8
import numpy as np
9
import numpy.typing as npt
@@ -27,7 +27,7 @@ class ProcessorInputs:
27
Represents the keyword arguments to
28
{meth}`vllm.multimodal.processing.BaseMultiModalProcessor.apply`.
29
"""
30
- prompt_text: str
30
+ prompt: Union[str, list[int]]
31
mm_data: MultiModalDataDict
32
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
33
@@ -75,7 +75,12 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
75
"in an upcoming release.")
76
77
seq_len = self.info.ctx.model_config.max_model_len
78
- return self.get_dummy_processor_inputs(seq_len, mm_counts).prompt_text
78
+
79
+ prompt = self.get_dummy_processor_inputs(seq_len, mm_counts).prompt
80
+ if not isinstance(prompt, str):
81
+ prompt = self.info.get_tokenizer().decode(prompt)
82
+
83
+ return prompt
84
85
# TODO: @abstractmethod after transition
86
def get_dummy_mm_data(
@@ -101,7 +106,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
106
dummy_text = self.get_dummy_text(mm_counts)
107
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)
108
104
- return ProcessorInputs(prompt_text=dummy_text, mm_data=dummy_mm_data)
109
+ return ProcessorInputs(prompt=dummy_text, mm_data=dummy_mm_data)
110
111
def _get_dummy_audios(
112
self,
@@ -177,7 +182,7 @@ class MultiModalProfiler(Generic[_I]):
182
seq_len, mm_counts)
183
184
return self.processor.apply(
180
- prompt=processor_inputs.prompt_text,
185
+ prompt=processor_inputs.prompt,
186
mm_data=processor_inputs.mm_data,
187
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
188
)