A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[CI/Build][Bugfix] Ensure compatibility with transformers 4.52 (#18678)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jun 4, 2025 at 19:49 UTC
01dc9a76db7d314aaf51be9ffc6ff561bae5626f
13 files changed
+82
-47
requirements/test.in
+1
-1
@@ -34,7 +34,7 @@ opencv-python-headless >= 4.11.0 # required for video test
34
datamodel_code_generator # required for minicpm3 test
35
lm-eval[api]==0.4.8 # required for model evaluation test
36
mteb>=1.38.11, <2 # required for mteb test
37
-transformers==4.51.3
37
+transformers==4.52.4
38
tokenizers==0.21.1
39
huggingface-hub[hf_xet]>=0.30.0 # Required for Xet downloads.
40
schemathesis>=3.39.15 # Required for openai schema test.
requirements/test.txt
+1
-1
@@ -794,7 +794,7 @@ tqdm==4.66.6
794
# transformers
795
tqdm-multiprocess==0.0.11
796
# via lm-eval
797
-transformers==4.51.3
797
+transformers==4.52.4
798
# via
799
# -r requirements/test.in
800
# genai-perf
tests/models/multimodal/generation/test_common.py
+7
-2
@@ -226,6 +226,8 @@ VLM_TEST_SETTINGS = {
226
img_idx_to_prompt=lambda idx: "",
227
auto_cls=AutoModelForImageTextToText,
228
vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output,
229
+ # FIXME: https://github.com/huggingface/transformers/pull/38510
230
+ marks=[pytest.mark.skip("Model is broken")],
231
),
232
"chameleon": VLMTestInfo(
233
models=["facebook/chameleon-7b"],
@@ -281,10 +283,10 @@ VLM_TEST_SETTINGS = {
283
multi_image_prompt="<start_of_image><start_of_image>Describe the two images in detail.", # noqa: E501
284
max_model_len=4096,
285
max_num_seqs=2,
284
- dtype="bfloat16",
286
auto_cls=AutoModelForImageTextToText,
287
vllm_runner_kwargs={"mm_processor_kwargs": {"do_pan_and_scan": True}},
288
patch_hf_runner=model_utils.gemma3_patch_hf_runner,
289
+ num_logprobs=10,
290
),
291
"glm4v": VLMTestInfo(
292
models=["THUDM/glm-4v-9b"],
@@ -337,7 +339,8 @@ VLM_TEST_SETTINGS = {
339
models=[
340
"OpenGVLab/InternVL2-1B",
341
"OpenGVLab/InternVL2-2B",
340
- "OpenGVLab/Mono-InternVL-2B",
342
+ # FIXME: Config cannot be loaded in transformers 4.52
343
+ # "OpenGVLab/Mono-InternVL-2B",
344
],
345
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
346
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
@@ -568,6 +571,8 @@ VLM_TEST_SETTINGS = {
571
max_num_seqs=2,
572
vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
573
prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
574
+ # FIXME: https://github.com/huggingface/transformers/issues/38358
575
+ marks=[pytest.mark.skip("Model initialization fails")],
576
),
577
"qwen2_vl": VLMTestInfo(
578
models=["Qwen/Qwen2-VL-2B-Instruct"],
tests/models/multimodal/generation/test_florence2.py
+2
@@ -100,6 +100,8 @@ def run_test(
100
)
101
102
103
+# FIXME: https://github.com/huggingface/transformers/issues/38358
104
+@pytest.mark.skip("Model initialization fails")
105
@pytest.mark.core_model
106
@pytest.mark.parametrize("model", MODELS)
107
@pytest.mark.parametrize(
tests/models/multimodal/generation/test_granite_speech.py
+1
-1
@@ -29,7 +29,7 @@ def vllm_to_hf_output(
29
return output_ids, hf_output_str, out_logprobs
30
31
32
-MODEL_NAME = "ibm-granite/granite-speech-3.3-8b"
32
+MODEL_NAME = "ibm-granite/granite-speech-3.3-2b"
33
# Audio lora co-exists directly in the model directory, but
34
# currently still needs to be passed directly to vLLM.
35
audio_lora_path = MODEL_NAME
tests/models/multimodal/generation/test_phi4mm.py
+4
@@ -122,6 +122,10 @@ def run_test(
122
for prompts, images, audios in inputs
123
]
124
125
+ # This error occurs inside `get_peft_model`
126
+ # FIXME: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/75
127
+ pytest.skip("HF impl is not compatible with current transformers")
128
+
129
hf_model_kwargs = {"_attn_implementation": "sdpa"}
130
with hf_runner(model, dtype=dtype,
131
model_kwargs=hf_model_kwargs) as hf_model:
tests/models/multimodal/generation/vlm_utils/model_utils.py
+17
-1
@@ -10,11 +10,12 @@ from typing import Optional, Union
10
11
import numpy as np
12
import numpy.typing as npt
13
+import pytest
14
import regex as re
15
import torch
16
from PIL.Image import Image
17
from transformers import (AutoConfig, AutoTokenizer, BatchFeature,
17
- GenerationConfig)
18
+ GenerationConfig, GenerationMixin)
19
20
from vllm.sequence import SampleLogprobs
21
from vllm.transformers_utils.tokenizer import patch_padding_side
@@ -324,6 +325,16 @@ def gemma3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
325
326
hf_model.processor = processor
327
328
+ orig_generate = hf_model.model.generate
329
+
330
+ def _generate(self, *args, **kwargs):
331
+ # FIXME: https://github.com/huggingface/transformers/issues/38333
332
+ kwargs["disable_compile"] = True
333
+
334
+ return orig_generate(*args, **kwargs)
335
+
336
+ hf_model.model.generate = types.MethodType(_generate, hf_model.model)
337
+
338
return hf_model
339
340
@@ -610,6 +621,11 @@ def _internvl_generate(
621
if getattr(self, "use_visual_token_mask", False):
622
visual_token_mask = selected.reshape(B, N, 1).to(input_embeds.dtype)
623
forward_kwargs["visual_token_mask"] = visual_token_mask
624
+
625
+ # e.g. InternVL2-2B
626
+ if not isinstance(self.language_model, GenerationMixin):
627
+ pytest.skip("HF impl is not compatible with current transformers")
628
+
629
outputs = self.language_model.generate(
630
**forward_kwargs,
631
**generate_kwargs,
tests/models/multimodal/processing/test_common.py
+1
-1
@@ -245,7 +245,7 @@ def _test_processing_correctness_one(
245
"adept/fuyu-8b",
246
"google/gemma-3-4b-it",
247
"THUDM/glm-4v-9b",
248
- "ibm-granite/granite-speech-3.3-8b",
248
+ "ibm-granite/granite-speech-3.3-2b",
249
"h2oai/h2ovl-mississippi-800m",
250
"OpenGVLab/InternVL2-1B",
251
"OpenGVLab/InternVL3-1B",
tests/models/registry.py
+15
-32
@@ -160,17 +160,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
160
"Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
161
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
162
"FalconH1ForCausalLM":_HfExamplesInfo("tiiuae/Falcon-H1-1.5B-Instruct",
163
- is_available_online=False,
164
- min_transformers_version="4.52.2"),
163
+ min_transformers_version="4.53"),
164
"GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
165
"Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
166
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
167
"GlmForCausalLM": _HfExamplesInfo("THUDM/glm-4-9b-chat-hf"),
169
- "Glm4ForCausalLM": _HfExamplesInfo(
170
- "THUDM/GLM-4-32B-0414",
171
- is_available_online=False,
172
- min_transformers_version="4.52.dev0"
173
- ),
168
+ "Glm4ForCausalLM": _HfExamplesInfo("THUDM/GLM-4-9B-0414"),
169
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2",
170
{"alias": "gpt2"}),
171
"GPTBigCodeForCausalLM": _HfExamplesInfo("bigcode/starcoder",
@@ -181,8 +176,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
176
{"1b": "EleutherAI/pythia-1.4b"}),
177
"GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
178
"GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
184
- "GraniteMoeHybridForCausalLM": _HfExamplesInfo("ibm-granite/granite-4.0-tiny-preview", # noqa: E501
185
- min_transformers_version="4.52.0"), # noqa: E501
179
+ "GraniteMoeHybridForCausalLM": _HfExamplesInfo("ibm-granite/granite-4.0-tiny-preview"), # noqa: E501
180
"GraniteMoeSharedForCausalLM": _HfExamplesInfo("ibm-research/moe-7b-1b-active-shared-experts"), # noqa: E501
181
"Grok1ModelForCausalLM": _HfExamplesInfo("hpcai-tech/grok-1",
182
trust_remote_code=True),
@@ -203,8 +197,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
197
"LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
198
is_available_online=False),
199
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
206
- "Mamba2ForCausalLM": _HfExamplesInfo("mistralai/Mamba-Codestral-7B-v0.1",
207
- is_available_online=False),
200
+ "Mamba2ForCausalLM": _HfExamplesInfo("mistralai/Mamba-Codestral-7B-v0.1"),
201
"FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"), # noqa: E501
202
"MiniCPMForCausalLM": _HfExamplesInfo("openbmb/MiniCPM-2B-sft-bf16",
203
trust_remote_code=True),
@@ -243,10 +236,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
236
"Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
237
"Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
238
"Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
246
- "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b",
247
- is_available_online=False),
239
+ "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
240
"StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b", # noqa: E501
249
- is_available_online=False),
241
+ v0_only=True),
242
"StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t",
243
v0_only=True),
244
"Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
@@ -256,7 +248,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
248
"TeleFLMForCausalLM": _HfExamplesInfo("CofeAI/FLM-2-52B-Instruct-2407",
249
trust_remote_code=True),
250
"XverseForCausalLM": _HfExamplesInfo("xverse/XVERSE-7B-Chat",
259
- is_available_online=False,
251
+ tokenizer="meta-llama/Llama-2-7b",
252
trust_remote_code=True),
253
"Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
254
"MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL",
@@ -275,8 +267,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
267
trust_remote_code=True),
268
"GteNewModel": _HfExamplesInfo("Alibaba-NLP/gte-base-en-v1.5",
269
trust_remote_code=True,
278
- hf_overrides={"architectures":
279
- ["GteNewModel"]}),
270
+ hf_overrides={"architectures": ["GteNewModel"]}), # noqa: E501
271
"InternLM2ForRewardModel": _HfExamplesInfo("internlm/internlm2-1_8b-reward",
272
trust_remote_code=True),
273
"JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"), # noqa: E501
@@ -298,10 +289,8 @@ _EMBEDDING_EXAMPLE_MODELS = {
289
"Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
290
trust_remote_code=True),
291
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"), # noqa: E501
301
- # The model on Huggingface is currently being updated,
302
- # hence I temporarily mark it as not available online
303
- "PrithviGeoSpatialMAE": _HfExamplesInfo("ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", # noqa: E501
304
- is_available_online=False),
292
+ "PrithviGeoSpatialMAE": _HfExamplesInfo("ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", # noqa: E501
293
+ is_available_online=False), # noqa: E501
294
}
295
296
_CROSS_ENCODER_EXAMPLE_MODELS = {
@@ -327,8 +316,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
316
hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]}), # noqa: E501
317
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
318
"Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
330
- "GraniteSpeechForConditionalGeneration": _HfExamplesInfo("ibm-granite/granite-speech-3.3-8b", # noqa: E501
331
- min_transformers_version="4.52.0"), # noqa: E501
319
+ "GraniteSpeechForConditionalGeneration": _HfExamplesInfo("ibm-granite/granite-speech-3.3-2b"), # noqa: E501
320
"GLM4VForCausalLM": _HfExamplesInfo("THUDM/glm-4v-9b",
321
trust_remote_code=True,
322
hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501
@@ -347,7 +335,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
335
trust_remote_code=True,
336
v0_only=True),
337
"Llama4ForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-4-Scout-17B-16E-Instruct", # noqa: E501
350
- min_transformers_version="4.51",
338
max_model_len=10240),
339
"LlavaForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-1.5-7b-hf",
340
extras={"mistral": "mistral-community/pixtral-12b", # noqa: E501
@@ -360,8 +347,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
347
transformers_version_reason="HF model is not compatible.", # noqa: E501
348
hf_overrides={"architectures": ["MantisForConditionalGeneration"]}), # noqa: E501
349
"MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6",
363
- max_transformers_version="4.48",
364
- transformers_version_reason="Use of deprecated imports which have been removed.", # noqa: E501
350
trust_remote_code=True),
351
"MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
352
extras={"2.6": "openbmb/MiniCPM-V-2_6"}, # noqa: E501
@@ -399,10 +384,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
384
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
385
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
386
"Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct"), # noqa: E501
402
- "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B",
403
- min_transformers_version="4.52"),
404
- "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ", # noqa: E501
405
- min_transformers_version="4.52"),
387
+ "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
388
+ "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), # noqa: E501
389
"SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),
390
"SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct"), # noqa: E501
391
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
@@ -413,8 +396,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
396
# Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
397
# Therefore, we borrow the BartTokenizer from the original Bart model
398
"Florence2ForConditionalGeneration": _HfExamplesInfo("microsoft/Florence-2-base", # noqa: E501
416
- tokenizer="Isotr0py/Florence-2-tokenizer",
417
- trust_remote_code=True,), # noqa: E501
399
+ tokenizer="Isotr0py/Florence-2-tokenizer", # noqa: E501
400
+ trust_remote_code=True), # noqa: E501
401
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
402
"WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
403
}
tests/models/test_initialization.py
+11
@@ -21,6 +21,10 @@ def test_can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch):
21
model_info.check_available_online(on_fail="skip")
22
model_info.check_transformers_version(on_fail="skip")
23
24
+ # FIXME: Possible memory leak in the previous tests?
25
+ if model_arch == "GraniteSpeechForConditionalGeneration":
26
+ pytest.skip("Avoid OOM")
27
+
28
# Avoid OOM and reduce initialization time by only using 1 layer
29
def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
30
hf_config.update(model_info.hf_overrides)
@@ -41,6 +45,13 @@ def test_can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch):
45
"num_hidden_layers": 1,
46
})
47
48
+ # e.g.: ibm-granite/granite-speech-3.3-2b
49
+ if hasattr(hf_config, "encoder_config"):
50
+ hf_config.encoder_config.update({
51
+ "num_layers": 1,
52
+ "num_hidden_layers": 1,
53
+ })
54
+
55
return hf_config
56
57
# Avoid calling model.forward()
vllm/config.py
+2
@@ -3139,6 +3139,8 @@ def _find_dtype(
3139
config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
3140
if config_dtype is None and hasattr(config, "vision_config"):
3141
config_dtype = getattr(config.vision_config, "torch_dtype", None)
3142
+ if config_dtype is None and hasattr(config, "encoder_config"):
3143
+ config_dtype = getattr(config.encoder_config, "torch_dtype", None)
3144
3145
# Try to read the dtype of the weights if they are in safetensors format
3146
if config_dtype is None:
vllm/model_executor/models/aya_vision.py
+8
-4
@@ -111,7 +111,13 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):
111
return self.ctx.get_hf_config(AyaVisionConfig)
112
113
def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor:
114
- return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
114
+ processor = self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
115
+
116
+ # Temporary workaround since this processor has multiple image tokens
117
+ # See https://github.com/huggingface/transformers/issues/38350
118
+ processor._check_special_mm_tokens = lambda *args, **kwargs: None
119
+
120
+ return processor
121
122
def get_image_processor(self) -> GotOcr2ImageProcessor:
123
return self.get_hf_processor().image_processor
@@ -188,9 +194,7 @@ class AyaVisionMultiModalProcessor(
194
image_processor = hf_processor.image_processor
195
196
# HF processor pops the `num_patches` kwarg, which is needed by vLLM
191
- if (images :=
192
- mm_data.get("images")) is not None and '<image>' in prompt:
193
- assert isinstance(images, list)
197
+ if (images := mm_data.get("images")) is not None:
198
parsed_images = (self._get_data_parser().parse_mm_data({
199
"image":
200
images
vllm/model_executor/models/idefics3.py
+12
-4
@@ -22,8 +22,8 @@ from typing import Literal, Optional, TypedDict, Union
22
23
import torch
24
from torch import nn
25
-from transformers import (BatchFeature, Idefics3Config, Idefics3ImageProcessor,
26
- Idefics3Processor)
25
+from transformers import (AddedToken, BatchFeature, Idefics3Config,
26
+ Idefics3ImageProcessor, Idefics3Processor)
27
28
from vllm.config import VllmConfig
29
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -199,13 +199,21 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
199
200
return grid_w * grid_h + 1
201
202
+ # TODO: Remove after requiring transformers>=4.52
203
+ def _get_content(self, token: Union[AddedToken, str]) -> str:
204
+ if isinstance(token, str):
205
+ return token
206
+
207
+ return token.content
208
+
209
def _get_image_token(
210
self,
211
processor: Optional[Idefics3Processor]) -> tuple[str, str, str]:
212
if processor is None:
213
processor = self.get_hf_processor()
207
- image_token = processor.image_token.content
208
- fake_image_token = processor.fake_image_token.content
214
+
215
+ image_token = self._get_content(processor.image_token)
216
+ fake_image_token = self._get_content(processor.fake_image_token)
217
global_image_token = processor.global_image_tag
218
return image_token, fake_image_token, global_image_token
219