01cae37713 · tduncan/Nvidia-Optimization-Stack-VLLM

requirements/test.in

+1 -1

@@ -34,7 +34,7 @@ opencv-python-headless >= 4.11.0 # required for video test

34 datamodel_code_generator # required for minicpm3 test

35 lm-eval[api]==0.4.8 # required for model evaluation test

36 mteb[bm25s]>=1.38.11, <2 # required for mteb test

37 -transformers==4.52.4

37 +transformers==4.53.2

38 tokenizers==0.21.1

39 huggingface-hub[hf_xet]>=0.33.0 # Required for Xet downloads.

40 schemathesis>=3.39.15 # Required for openai schema test.

requirements/test.txt

+1 -1

@@ -800,7 +800,7 @@ tqdm==4.66.6

800 # transformers

801 tqdm-multiprocess==0.0.11

802 # via lm-eval

803 -transformers==4.52.4

803 +transformers==4.53.2

804 # via

805 # -r requirements/test.in

806 # genai-perf

tests/models/multimodal/generation/test_common.py

+2 -2

@@ -318,6 +318,7 @@ VLM_TEST_SETTINGS = {

318 num_logprobs=10,

319 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],

320 auto_cls=AutoModelForImageTextToText,

321 + marks=[large_gpu_mark(min_gb=32)],

322 ),

323 "glm4_1v-video": VLMTestInfo(

324 models=["THUDM/GLM-4.1V-9B-Thinking"],

@@ -331,8 +332,7 @@ VLM_TEST_SETTINGS = {

332 inputs=custom_inputs.video_with_metadata_glm4_1v(),

333 limit_mm_per_prompt={"video": 1},

334 )],

334 - # This is needed to run on machine with 24GB VRAM

335 - vllm_runner_kwargs={"gpu_memory_utilization": 0.95},

335 + marks=[large_gpu_mark(min_gb=32)],

336 ),

337 "h2ovl": VLMTestInfo(

338 models = [

tests/models/multimodal/processing/test_common.py

+1

@@ -159,6 +159,7 @@ def _test_processing_correctness(

159 _ADD_SPECIAL_TOKENS_OVERRIDES = {

160 "mllama": False,

161 "ovis": False,

162 + "paligemma": False,

163 "ultravox": False,

164 "whisper": False,

165 }

tests/models/test_initialization.py

+10 -2

@@ -31,7 +31,8 @@ def test_can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch):

31 model_info.check_transformers_version(on_fail="skip")

32

33 # FIXME: Possible memory leak in the previous tests?

34 - if model_arch in ("GraniteSpeechForConditionalGeneration",

34 + if model_arch in ("Glm4vForConditionalGeneration",

35 + "GraniteSpeechForConditionalGeneration",

36 "KimiVLForConditionalGeneration"):

37 pytest.skip("Avoid OOM")

38

@@ -46,9 +47,14 @@ def test_can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch):

47 n_group = getattr(text_config, 'n_group', None)

48 num_experts = n_group * 2 if n_group is not None else 2

49

50 + # we use three layers for Gemma-3n to check

51 + # both normal layer and kv_shared_layer

52 + num_hidden_layers = (3 if model_arch

53 + == "Gemma3nForConditionalGeneration" else 1)

54 +

55 text_config.update({

56 "num_layers": 1,

51 - "num_hidden_layers": 1,

57 + "num_hidden_layers": num_hidden_layers,

58 "num_experts": num_experts,

59 "num_experts_per_tok": 2,

60 "num_local_experts": num_experts,

@@ -56,6 +62,8 @@ def test_can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch):

62 "first_k_dense_replace": 0,

63 # To avoid OOM on DeepSeek-V3

64 "n_routed_experts": num_experts,

65 + # For Gemma-3n

66 + "num_kv_shared_layers": 1,

67 })

68

69 if hasattr(hf_config, "vision_config"):

vllm/inputs/registry.py

+1 -7

@@ -5,9 +5,7 @@ from dataclasses import dataclass

5 from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union

6

7 import torch

8 -from packaging.version import Version

8 from transformers import BatchFeature, PretrainedConfig, ProcessorMixin

10 -from transformers import __version__ as TRANSFORMERS_VERSION

9 from typing_extensions import TypeVar

10

11 from vllm.jsontree import JSONTree, json_map_leaves

@@ -137,13 +135,9 @@ class InputProcessingContext(InputContext):

135 /,

136 **kwargs: object,

137 ) -> _P:

140 - # Transformers 4.53.0 has issue with passing tokenizer to

141 - # initialize processor. We disable it for this version.

142 - # See: https://github.com/vllm-project/vllm/issues/20224

143 - if Version(TRANSFORMERS_VERSION) != Version("4.53.0"):

144 - kwargs["tokenizer"] = self.tokenizer

138 return super().get_hf_processor(

139 typ,

140 + tokenizer=self.tokenizer,

141 **kwargs,

142 )

143

vllm/model_executor/models/commandr.py

+5 -2

@@ -189,10 +189,13 @@ class CohereAttention(nn.Module):

189

190 layer_idx = extract_layer_index(prefix)

191 layer_has_sliding_window = (

192 - getattr(config, "sliding_window_pattern", False)

193 - and (layer_idx + 1) % self.config.sliding_window_pattern != 0)

192 + getattr(config, "sliding_window_pattern", False) and

193 + (layer_idx + 1) % self.config.sliding_window_pattern

194 + != 0) or (getattr(config, "layer_types", False)

195 + and config.layer_types[layer_idx] == "sliding_attention")

196

197 self.sliding_window = (interleaved_sliding_window

198 + or config.sliding_window

199 if layer_has_sliding_window else None)

200

201 self.attn = Attention(self.num_heads,

vllm/model_executor/models/fuyu.py

+18 -7

@@ -175,12 +175,21 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):

175

176 # Original output: (1, num_images, Pn, Px * Py * C)

177 # New output: (num_images, Pn, Px * Py * C)

178 - assert (isinstance(image_patches, list)

179 - and len(image_patches) == 1)

180 - assert (isinstance(image_patches[0], torch.Tensor)

181 - and len(image_patches[0]) == len(images))

182 -

183 - processed_outputs["image_patches"] = image_patches[0]

178 + # image_patches is a list with shape:

179 + # (1, num_images, Pn, Px * Py * C)

180 + # before Transformers 4.53

181 + if isinstance(image_patches, list):

182 + assert len(image_patches) == 1

183 + assert (isinstance(image_patches[0], torch.Tensor)

184 + and len(image_patches[0]) == len(images))

185 + processed_outputs["image_patches"] = image_patches[0]

186 + # image_patches is a tensor with shape:

187 + # (num_images, Pn, Px * Py * C)

188 + # after Transformers 4.53

189 + elif isinstance(image_patches, torch.Tensor):

190 + assert len(image_patches) == len(images)

191 + else:

192 + raise AssertionError("This line should be unreachable.")

193

194 return processed_outputs

195

@@ -193,8 +202,10 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):

202 vocab = tokenizer.get_vocab()

203

204 boa_token_id = vocab["<0x04>"]

205 + if prompt_tokens[-1] != boa_token_id:

206 + prompt_tokens.append(boa_token_id)

207

197 - return prompt_tokens + [boa_token_id]

208 + return prompt_tokens

209

210 def _get_mm_fields_config(

211 self,

vllm/model_executor/models/gemma3.py

+6 -3

@@ -149,14 +149,17 @@ class Gemma3Attention(nn.Module):

149 # TODO(woosuk): Add reference to the original HF implementation.

150 layer_idx = extract_layer_index(prefix)

151 self.is_sliding = (getattr(

152 - config, "interleaved_sliding_window", None) is not None and bool(

153 - (layer_idx + 1) % config.sliding_window_pattern))

152 + config, "interleaved_sliding_window", None) is not None and (bool(

153 + (layer_idx + 1) % config.sliding_window_pattern))) or (

154 + getattr(config, "layer_types", None) is not None

155 + and config.layer_types[layer_idx] == "sliding_attention")

156 # Initialize the rotary embedding.

157 if self.is_sliding:

158 # Local attention. Override the values in config.json.

159 self.rope_theta = config.rope_local_base_freq

160 self.rope_scaling = {"rope_type": "default"}

159 - self.sliding_window = config.interleaved_sliding_window

161 + self.sliding_window = (config.interleaved_sliding_window

162 + or config.sliding_window)

163 else:

164 # Global attention. Use the values in config.json.

165 self.rope_theta = config.rope_theta

vllm/model_executor/models/minicpmo.py

+11 -10

@@ -30,8 +30,10 @@ import torch

30 from torch import nn

31 from transformers import BatchFeature, PretrainedConfig

32 from transformers.modeling_outputs import BaseModelOutputWithPast

33 -from transformers.models.whisper.modeling_whisper import (

34 - ACT2FN, WHISPER_ATTENTION_CLASSES, WhisperConfig, WhisperEncoder)

33 +from transformers.models.whisper.modeling_whisper import (ACT2FN,

34 + WhisperAttention,

35 + WhisperConfig,

36 + WhisperEncoder)

37

38 from vllm.config import VllmConfig

39 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -378,14 +380,13 @@ class MiniCPMWhisperEncoderLayer(nn.Module):

380 def __init__(self, config: WhisperConfig, layer_idx: int):

381 super().__init__()

382 self.embed_dim = config.d_model

381 - self.self_attn = WHISPER_ATTENTION_CLASSES[

382 - config._attn_implementation](

383 - embed_dim=self.embed_dim,

384 - num_heads=config.encoder_attention_heads,

385 - dropout=config.attention_dropout,

386 - config=config,

387 - layer_idx=layer_idx,

388 - )

383 + self.self_attn = WhisperAttention(

384 + embed_dim=self.embed_dim,

385 + num_heads=config.encoder_attention_heads,

386 + dropout=config.attention_dropout,

387 + config=config,

388 + layer_idx=layer_idx,

389 + )

390 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)

391 self.dropout = config.dropout

392 self.activation_fn = ACT2FN[config.activation_function]

vllm/model_executor/models/paligemma.py

+1 -1

@@ -125,7 +125,7 @@ class PaliGemmaMultiModalProcessor(

125 ) -> BatchFeature:

126 tokenizer = self.info.get_tokenizer()

127 if not mm_data:

128 - prompt_ids = tokenizer.encode(prompt)

128 + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)

129 return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")

130

131 return super()._call_hf_processor(

vllm/model_executor/models/qwen2_5_omni_thinker.py

+9 -1

@@ -144,8 +144,16 @@ class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo,

144 ) -> Qwen2_5OmniProcessor:

145 if fps is not None:

146 kwargs["fps"] = fps

147 +

148 + # Monkey patch for Transformers v4.53

149 + processor_class = Qwen2_5OmniProcessor

150 + if processor_class.image_processor_class != "AutoImageProcessor":

151 + processor_class.image_processor_class = "AutoImageProcessor"

152 + if processor_class.video_processor_class != "AutoVideoProcessor":

153 + processor_class.video_processor_class = "AutoVideoProcessor"

154 +

155 processor = self.ctx.get_hf_processor(

148 - Qwen2_5OmniProcessor,

156 + processor_class,

157 image_processor=self.get_image_processor(min_pixels=min_pixels,

158 max_pixels=max_pixels,

159 size=size,

vllm/model_executor/models/whisper.py

+8 -1

@@ -634,7 +634,14 @@ class WhisperProcessingInfo(BaseProcessingInfo):

634 def get_hf_processor(self,

635 sampling_rate: Optional[int] = None

636 ) -> WhisperProcessor:

637 - return self.ctx.get_hf_processor(WhisperProcessor)

637 + # HACK: Transformers 4.53.0 has issue with whisper tokenizer to

638 + # initialize processor. We use a monkeypatch to fix it here.

639 + # See: https://github.com/vllm-project/vllm/issues/20224

640 + processor_class = WhisperProcessor

641 + tokenizer_class = ("WhisperTokenizer", "WhisperTokenizerFast")

642 + if processor_class.tokenizer_class != tokenizer_class:

643 + processor_class.tokenizer_class = tokenizer_class

644 + return self.ctx.get_hf_processor(processor_class)

645

646 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:

647 return {"audio": 1}

[CI/Build] Ensure compatability with Transformers v4.53 (#20541)