A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 17:45 UTC
[Bugfix] Fix models and tests for transformers v5 (#33977)
Signed-off-by: raushan <raushan@huggingface.co> Signed-off-by: Raushan Turganbay <raushan.turganbay@alumni.nu.edu.kz> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Raushan Turganbay committed
Feb 6, 2026 at 14:47 UTC
85ee1d962b1339eb597bf9451a0bcf4b91c72f2a
12 files changed
+70
-55
docs/models/supported_models.md
+1
-1
@@ -674,7 +674,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
674
| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
675
|--------------|--------|--------|-------------------|----------------------|---------------------------|
676
| `AriaForConditionalGeneration` | Aria | T + I<sup>+</sup> | `rhymes-ai/Aria` | | |
677
-| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A<sup>+</sup> | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-2601-hf` | ✅︎ | ✅︎ |
677
+| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-2601-hf` | ✅︎ | ✅︎ |
678
| `AyaVisionForConditionalGeneration` | Aya Vision | T + I<sup>+</sup> | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |
679
| `BagelForConditionalGeneration` | BAGEL | T + I<sup>+</sup> | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
680
| `BeeForConditionalGeneration` | Bee-8B | T + I<sup>E+</sup> | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
tests/models/multimodal/generation/test_qwen2_vl.py
+4
-4
@@ -198,10 +198,10 @@ def batch_make_video_embeddings(
198
videos += video_batch
199
200
# video to pixel values
201
- image_processor = processor.image_processor
201
+ video_processor = processor.video_processor
202
203
- preprocess_result = image_processor.preprocess(
204
- images=None, videos=videos, return_tensors="pt"
203
+ preprocess_result = video_processor.preprocess(
204
+ videos=videos, return_tensors="pt"
205
).data
206
pixel_values = preprocess_result["pixel_values_videos"]
207
video_grid_thw = preprocess_result["video_grid_thw"]
@@ -222,7 +222,7 @@ def batch_make_video_embeddings(
222
embed_counter = 0
223
for video_batch in video_batches_:
224
cur_batch_video_count = len(video_batch)
225
- merge_size = image_processor.merge_size
225
+ merge_size = video_processor.merge_size
226
cur_batch_embed_len = sum(
227
grid_thw.prod(-1) // merge_size // merge_size
228
for grid_thw in video_grid_thw[
tests/models/multimodal/pooling/test_llava_next.py
+1
-1
@@ -81,7 +81,7 @@ def _run_test(
81
# Patch the issue where image_token_id
82
# exceeds the maximum allowed vocab size
83
hf_model.model.resize_token_embeddings(
84
- hf_model.model.language_model.vocab_size + 1
84
+ hf_model.model.model.language_model.vocab_size + 1
85
)
86
87
all_inputs = hf_model.get_inputs(input_texts, images=input_images)
tests/models/multimodal/test_mapping.py
+3
-1
@@ -33,7 +33,9 @@ def create_dummy_model(repo: str, model_arch: str) -> PreTrainedModel:
33
model = model_cls._from_config(config)
34
# TODO(hmellor): Remove this once Transformers has fixed tied weights on meta device
35
# https://github.com/huggingface/transformers/issues/43522
36
- if getattr(config.get_text_config(), "tie_word_embeddings", False):
36
+ if getattr(config.get_text_config(), "tie_word_embeddings", False) or getattr(
37
+ config, "tie_word_embeddings", False
38
+ ):
39
model.tie_weights()
40
return model
41
vllm/model_executor/models/audioflamingo3.py
+1
-1
@@ -236,7 +236,7 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
236
)
237
238
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
239
- return {"audio": None}
239
+ return {"audio": 1}
240
241
242
class AudioFlamingo3DummyInputsBuilder(
vllm/model_executor/models/hunyuan_vision.py
+1
-1
@@ -692,7 +692,7 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
692
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
693
num_images = mm_counts.get("image", 0)
694
695
- hf_processor = self.info.get_hf_processor()
695
+ hf_processor = self.info.get_hf_processor(typ=HunYuanVLProcessor)
696
image_token: str = hf_processor.image_token
697
698
return image_token * num_images
vllm/model_executor/models/isaac.py
+1
-1
@@ -13,7 +13,7 @@ import torch.nn as nn
13
import torch.nn.functional as F
14
from einops import rearrange
15
from transformers.image_processing_utils import BatchFeature
16
-from transformers.tokenization_utils import TensorType
16
+from transformers.utils import TensorType
17
from typing_extensions import TypedDict, Unpack
18
19
from vllm.config import VllmConfig
vllm/model_executor/models/minimax_vl_01.py
+2
-2
@@ -230,8 +230,8 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
230
self.vision_feature_layer = config.vision_feature_layer
231
self.vocab_size = config.text_config.vocab_size
232
self.pad_token_id = -1
233
- if self.config.pad_token_id is not None:
234
- self.pad_token_id = self.config.pad_token_id
233
+ if self.config.text_config.pad_token_id is not None:
234
+ self.pad_token_id = self.config.text_config.pad_token_id
235
236
self.make_empty_intermediate_tensors = (
237
self.language_model.make_empty_intermediate_tensors
vllm/transformers_utils/processors/bagel.py
+26
-20
@@ -6,10 +6,18 @@
6
from transformers import AutoProcessor
7
from transformers.feature_extraction_utils import BatchFeature
8
from transformers.image_utils import ImageInput
9
-from transformers.processing_utils import ProcessorMixin
9
+from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
10
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
11
12
13
+class BagelProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
14
+ _defaults = {
15
+ "images_kwargs": {
16
+ "return_tensors": "pt",
17
+ },
18
+ }
19
+
20
+
21
class BagelProcessor(ProcessorMixin):
22
"""
23
Constructs a BAGEL processor which wraps a
@@ -27,34 +35,32 @@ class BagelProcessor(ProcessorMixin):
35
| list[TextInput]
36
| list[PreTokenizedInput] = None,
37
images: ImageInput = None,
30
- **kwargs,
38
+ **kwargs: Unpack[BagelProcessorKwargs],
39
):
40
"""
41
Main method to prepare for the model one or several sequences(s) and image(s).
42
"""
43
+ output_kwargs = self._merge_kwargs(
44
+ BagelProcessorKwargs,
45
+ tokenizer_init_kwargs=self.tokenizer.init_kwargs,
46
+ **kwargs,
47
+ )
48
+
49
if images is not None:
50
# Process images with the image processor
37
- # Ensure return_tensors is set to "pt" for PyTorch tensors
38
- image_kwargs = {**kwargs}
39
- if "return_tensors" not in image_kwargs:
40
- image_kwargs["return_tensors"] = "pt"
41
- pixel_values = self.image_processor(images, **image_kwargs)
51
+ pixel_values = self.image_processor(
52
+ images, **output_kwargs["images_kwargs"]
53
+ )
54
else:
43
- pixel_values = None
55
+ pixel_values = {}
56
45
- text_inputs = self.tokenizer(text, **kwargs) if text is not None else None
57
+ text_inputs = (
58
+ self.tokenizer(text, **output_kwargs["text_kwargs"])
59
+ if text is not None
60
+ else {}
61
+ )
62
47
- if pixel_values is not None and text_inputs is not None:
48
- # Combine text and image inputs into BatchFeature
49
- combined = dict(text_inputs)
50
- combined["pixel_values"] = pixel_values["pixel_values"]
51
- return BatchFeature(combined)
52
- elif pixel_values is not None:
53
- return pixel_values
54
- elif text_inputs is not None:
55
- return BatchFeature(dict(text_inputs))
56
- else:
57
- return BatchFeature({})
63
+ return BatchFeature(data={**pixel_values, **text_inputs})
64
65
def batch_decode(self, *args, **kwargs):
66
"""
vllm/transformers_utils/processors/hunyuan_vl.py
+1
-4
@@ -23,7 +23,6 @@ class HunYuanVLProcessor(ProcessorMixin):
23
self,
24
image_processor=None,
25
tokenizer=None,
26
- video_processor=None,
26
chat_template=None,
27
**kwargs,
28
):
@@ -42,9 +41,7 @@ class HunYuanVLProcessor(ProcessorMixin):
41
)
42
self.pad_id = 120002 # self.tokenizer.pad_token_id
43
45
- super().__init__(
46
- image_processor, tokenizer, video_processor, chat_template=chat_template
47
- )
44
+ super().__init__(image_processor, tokenizer, chat_template=chat_template)
45
46
def __call__(
47
self,
vllm/transformers_utils/processors/ovis.py
+11
-6
@@ -43,9 +43,7 @@ class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-a
43
"padding": False,
44
},
45
"images_kwargs": {
46
- "max_partition": 9,
47
- "covering_threshold": 0.9,
48
- "convert_to_rgb": True,
46
+ "do_convert_rgb": True,
47
"return_tensors": "pt",
48
},
49
}
@@ -143,6 +141,10 @@ class OvisProcessor(ProcessorMixin):
141
- **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`.
142
- **second_per_grid_ts** -- List of video seconds per time grid. Returned when `videos` is not `None`.
143
"""
144
+
145
+ max_partition = kwargs.pop("max_partition", 9)
146
+ covering_threshold = kwargs.pop("covering_threshold", 0.9)
147
+
148
output_kwargs = self._merge_kwargs(
149
OvisProcessorKwargs,
150
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
@@ -159,7 +161,10 @@ class OvisProcessor(ProcessorMixin):
161
# Process each image
162
for image in images if isinstance(images, list) else [images]:
163
pixel_values, image_placeholders, grid = self.preprocess_image(
162
- image=image, **output_kwargs["images_kwargs"]
164
+ image=image,
165
+ max_partition=max_partition,
166
+ covering_threshold=covering_threshold,
167
+ **output_kwargs["images_kwargs"],
168
)
169
processed_images.append(pixel_values)
170
image_placeholders_list.append(image_placeholders)
@@ -300,7 +305,7 @@ class OvisProcessor(ProcessorMixin):
305
image: PIL.Image.Image,
306
max_partition,
307
covering_threshold,
303
- convert_to_rgb,
308
+ do_convert_rgb,
309
return_tensors,
310
):
311
def _preprocess(img: PIL.Image.Image, side):
@@ -394,7 +399,7 @@ class OvisProcessor(ProcessorMixin):
399
# pick the partition with maximum covering_ratio and break the tie using #sub_images
400
return sorted(all_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0][0]
401
397
- if convert_to_rgb:
402
+ if do_convert_rgb:
403
image = convert_image_mode(image, "RGB")
404
405
sides = self.get_image_size()
vllm/transformers_utils/processors/ovis2_5.py
+18
-13
@@ -24,14 +24,10 @@ class Ovis2_5ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[cal
24
"padding": False,
25
},
26
"images_kwargs": {
27
- "convert_to_rgb": True,
28
- "min_pixels": MIN_PIXELS,
29
- "max_pixels": MAX_PIXELS,
27
+ "do_convert_rgb": True,
28
},
29
"videos_kwargs": {
32
- "convert_to_rgb": True,
33
- "min_pixels": MIN_PIXELS,
34
- "max_pixels": MAX_PIXELS,
30
+ "do_convert_rgb": True,
31
},
32
}
33
@@ -160,6 +156,9 @@ class Ovis2_5Processor(ProcessorMixin):
156
- **second_per_grid_ts** -- list of video seconds per time grid.
157
Returned when `videos` is not `None`.
158
"""
159
+ min_pixels = kwargs.pop("min_pixels", MIN_PIXELS)
160
+ max_pixels = kwargs.pop("max_pixels", MAX_PIXELS)
161
+
162
output_kwargs = self._merge_kwargs(
163
Ovis2_5ProcessorKwargs,
164
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
@@ -175,7 +174,10 @@ class Ovis2_5Processor(ProcessorMixin):
174
# Process each image
175
for image in images if isinstance(images, list) else [images]:
176
pixel_values, image_placeholders, grid = self.preprocess_multidata(
178
- images=image, **output_kwargs["images_kwargs"]
177
+ images=image,
178
+ min_pixels=min_pixels,
179
+ max_pixels=max_pixels,
180
+ **output_kwargs["images_kwargs"],
181
)
182
processed_images.append(pixel_values)
183
image_placeholders_list.append(image_placeholders)
@@ -194,7 +196,10 @@ class Ovis2_5Processor(ProcessorMixin):
196
# Process each video
197
for video in videos if isinstance(videos, list) else [videos]:
198
pixel_values, video_placeholders, grid = self.preprocess_multidata(
197
- video=video, **output_kwargs["videos_kwargs"]
199
+ video=video,
200
+ min_pixels=min_pixels,
201
+ max_pixels=max_pixels,
202
+ **output_kwargs["videos_kwargs"],
203
)
204
processed_videos.append(pixel_values)
205
videos_placeholders_list.append(video_placeholders)
@@ -378,7 +383,7 @@ class Ovis2_5Processor(ProcessorMixin):
383
self,
384
images: PIL.Image.Image | list[PIL.Image.Image] | None = None,
385
video: list[PIL.Image.Image] | np.ndarray | None = None,
381
- convert_to_rgb: bool | None = True,
386
+ do_convert_rgb: bool | None = True,
387
min_pixels: int = MIN_PIXELS,
388
max_pixels: int = MAX_PIXELS,
389
return_tensors: str | None = "pt",
@@ -404,7 +409,7 @@ class Ovis2_5Processor(ProcessorMixin):
409
min_pixels if min_pixels is not None else MIN_PIXELS,
410
)
411
images = [
407
- image.convert("RGB") if convert_to_rgb and image.mode != "RGB" else image
412
+ image.convert("RGB") if do_convert_rgb and image.mode != "RGB" else image
413
for image in images
414
]
415
@@ -420,9 +425,9 @@ class Ovis2_5Processor(ProcessorMixin):
425
max_pixels=max_pixels,
426
)
427
new_size = dict(height=resized_height, width=resized_width)
423
- image_pt = self.image_processor.preprocess(
424
- image, size=new_size, return_tensors="np"
425
- )["pixel_values"][0]
428
+ image_pt = self.image_processor.preprocess(image, size=new_size)[
429
+ "pixel_values"
430
+ ][0]
431
432
processed_images.append(image_pt)
433