[MM] Keep more metadata tensors on CPU (#52827)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Aug 18, 2026 at 19:56 UTC
deeeae75d0e42199aaeb627ad66e38b28337d2bf
38 files changed
+148
-83
vllm/model_executor/models/cohere2_vision.py
+1
-1
@@ -268,7 +268,7 @@ class Cohere2VisionMultiModalProcessor(
268
num_patches = hf_inputs.get("num_patches", torch.empty(0))
269
return dict(
270
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
271
- num_patches=MultiModalFieldConfig.batched("image"),
271
+ num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
272
image_embeds=MultiModalFieldConfig.batched("image"),
273
)
274
vllm/model_executor/models/deepseek_ocr.py
+3
-1
@@ -320,7 +320,9 @@ class DeepseekOCRMultiModalProcessor(
320
patches_per_image = torch.where(is_tiled, images_spatial_crop.prod(dim=-1), 0)
321
return dict(
322
pixel_values=MultiModalFieldConfig.batched("image"),
323
- images_spatial_crop=MultiModalFieldConfig.batched("image"),
323
+ images_spatial_crop=MultiModalFieldConfig.batched(
324
+ "image", keep_on_cpu=True
325
+ ),
326
images_crop=MultiModalFieldConfig.flat_from_sizes(
327
"image", patches_per_image
328
),
vllm/model_executor/models/deepseek_ocr2.py
+3
-1
@@ -191,7 +191,9 @@ class DeepseekOCR2MultiModalProcessor(
191
patches_per_image = torch.where(is_tiled, images_spatial_crop.prod(dim=-1), 0)
192
return dict(
193
pixel_values=MultiModalFieldConfig.batched("image"),
194
- images_spatial_crop=MultiModalFieldConfig.batched("image"),
194
+ images_spatial_crop=MultiModalFieldConfig.batched(
195
+ "image", keep_on_cpu=True
196
+ ),
197
images_crop=MultiModalFieldConfig.flat_from_sizes(
198
"image", patches_per_image
199
),
vllm/model_executor/models/deepseek_vl2.py
+3
-1
@@ -270,7 +270,9 @@ class DeepseekVL2MultiModalProcessor(
270
271
return dict(
272
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
273
- images_spatial_crop=MultiModalFieldConfig.batched("image"),
273
+ images_spatial_crop=MultiModalFieldConfig.batched(
274
+ "image", keep_on_cpu=True
275
+ ),
276
image_embeds=MultiModalFieldConfig.batched("image"),
277
)
278
vllm/model_executor/models/ernie45_vl.py
+2
-2
@@ -1246,11 +1246,11 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
1246
pixel_values=MultiModalFieldConfig.flat_from_sizes(
1247
"image", image_grid_sizes
1248
),
1249
- image_grid_thw=MultiModalFieldConfig.batched("image"),
1249
+ image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
1250
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
1251
"video", video_grid_sizes
1252
),
1253
- video_grid_thw=MultiModalFieldConfig.batched("video"),
1253
+ video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
1254
)
1255
1256
vllm/model_executor/models/fireredasr2.py
+1
-1
@@ -260,7 +260,7 @@ class FireRedASR2MultiModalProcessor(
260
return dict(
261
input_features=MultiModalFieldConfig.batched("audio"),
262
speech_lengths=MultiModalFieldConfig.batched("audio"),
263
- fake_token_lengths=MultiModalFieldConfig.batched("audio"),
263
+ fake_token_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
264
)
265
266
def _get_prompt_updates(
vllm/model_executor/models/fireredlid.py
+2
-2
@@ -482,8 +482,8 @@ class FireRedLIDMultiModalProcessor(
482
) -> Mapping[str, MultiModalFieldConfig]:
483
return dict(
484
input_features=MultiModalFieldConfig.batched("audio"),
485
- speech_lengths=MultiModalFieldConfig.batched("audio"),
486
- fake_token_lengths=MultiModalFieldConfig.batched("audio"),
485
+ speech_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
486
+ fake_token_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
487
)
488
489
def _get_prompt_updates(
vllm/model_executor/models/funasr.py
+1
-1
@@ -779,7 +779,7 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
779
return dict(
780
input_features=MultiModalFieldConfig.batched("audio"),
781
speech_lengths=MultiModalFieldConfig.batched("audio"),
782
- fake_token_lengths=MultiModalFieldConfig.batched("audio"),
782
+ fake_token_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
783
)
784
785
def _get_prompt_updates(
vllm/model_executor/models/funaudiochat.py
+3
-1
@@ -695,7 +695,9 @@ class FunAudioChatMultiModalProcessor(
695
) -> Mapping[str, MultiModalFieldConfig]:
696
return {
697
"speech_ids": MultiModalFieldConfig.batched("audio"),
698
- "speech_attention_mask": MultiModalFieldConfig.batched("audio"),
698
+ "speech_attention_mask": MultiModalFieldConfig.batched(
699
+ "audio", keep_on_cpu=True
700
+ ),
701
"input_features": MultiModalFieldConfig.batched("audio"),
702
"feature_attention_mask": MultiModalFieldConfig.batched("audio"),
703
"feature_exist_mask": MultiModalFieldConfig.batched("audio"),
vllm/model_executor/models/glmasr.py
+2
-2
@@ -625,13 +625,13 @@ def _glmasr_field_config(
625
feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
626
"audio", chunk_counts, dim=0
627
),
628
- chunk_counts=MultiModalFieldConfig.batched("audio"),
628
+ chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
629
)
630
return dict(
631
audio_embeds=MultiModalFieldConfig.batched("audio"),
632
input_features=MultiModalFieldConfig.batched("audio"),
633
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
634
- chunk_counts=MultiModalFieldConfig.batched("audio"),
634
+ chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
635
)
636
637
vllm/model_executor/models/granite4_vision.py
+1
-1
@@ -398,7 +398,7 @@ class Granite4VisionMultiModalProcessor(
398
) -> Mapping[str, MultiModalFieldConfig]:
399
return dict(
400
pixel_values=MultiModalFieldConfig.batched("image"),
401
- image_sizes=MultiModalFieldConfig.batched("image"),
401
+ image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
402
)
403
404
vllm/model_executor/models/hyperclovax_vision.py
+7
-3
@@ -324,10 +324,14 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
324
) -> Mapping[str, MultiModalFieldConfig]:
325
fields = dict(
326
pixel_values_images=MultiModalFieldConfig.batched("image"),
327
- image_sizes_images=MultiModalFieldConfig.batched("image"),
328
- vision_query_lengths_images=MultiModalFieldConfig.batched("image"),
327
+ image_sizes_images=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
328
+ vision_query_lengths_images=MultiModalFieldConfig.batched(
329
+ "image", keep_on_cpu=True
330
+ ),
331
pixel_values_videos=MultiModalFieldConfig.batched("video"),
330
- vision_query_lengths_videos=MultiModalFieldConfig.batched("video"),
332
+ vision_query_lengths_videos=MultiModalFieldConfig.batched(
333
+ "video", keep_on_cpu=True
334
+ ),
335
)
336
337
return fields
vllm/model_executor/models/isaac.py
+1
-1
@@ -440,7 +440,7 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
440
"pixel_values": MultiModalFieldConfig.flat_from_sizes(
441
"image", image_grid_sizes
442
),
443
- "image_grid_thw": MultiModalFieldConfig.batched("image"),
443
+ "image_grid_thw": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
444
}
445
446
def _get_prompt_updates(
vllm/model_executor/models/kanana_v.py
+2
-2
@@ -561,8 +561,8 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
561
562
mm_fields_config = dict(
563
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", pixel_sizes),
564
- vision_grid_thw=MultiModalFieldConfig.batched("image"),
565
- image_token_thw=MultiModalFieldConfig.batched("image"),
564
+ vision_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
565
+ image_token_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
566
)
567
return mm_fields_config
568
vllm/model_executor/models/keye_vl1_5.py
+5
-3
@@ -309,13 +309,15 @@ def _keye_field_config(
309
return dict(
310
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
311
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
312
- image_grid_thw=MultiModalFieldConfig.batched("image"),
312
+ image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
313
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
314
"video", video_num_patches
315
),
316
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_num_patches),
317
- video_grid_thw=MultiModalFieldConfig.flat_from_sizes("video", video_num_grids),
318
- num_frames=MultiModalFieldConfig.batched("video"),
317
+ video_grid_thw=MultiModalFieldConfig.flat_from_sizes(
318
+ "video", video_num_grids, keep_on_cpu=True
319
+ ),
320
+ num_frames=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
321
)
322
323
vllm/model_executor/models/kimi_audio.py
+1
-1
@@ -201,7 +201,7 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo
201
# Field config for Kimi-Audio multimodal data
202
_KIMIAUDIO_FIELD_CONFIG = {
203
"whisper_input_features": MultiModalFieldConfig.batched("audio"),
204
- "feature_attention_mask": MultiModalFieldConfig.batched("audio"),
204
+ "feature_attention_mask": MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
205
}
206
207
vllm/model_executor/models/kimi_vl.py
+1
-1
@@ -251,7 +251,7 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
251
pixel_values=MultiModalFieldConfig.flat_from_sizes(
252
"image", image_grid_sizes
253
),
254
- image_grid_hws=MultiModalFieldConfig.batched("image"),
254
+ image_grid_hws=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
255
)
256
257
def _get_prompt_updates(
vllm/model_executor/models/llava_onevision2.py
+4
-4
@@ -431,20 +431,20 @@ def _create_field_factory(
431
image_embeds=MultiModalFieldConfig.flat_from_sizes(
432
"image", image_embed_grid_sizes
433
),
434
- image_grid_thw=MultiModalFieldConfig.batched("image"),
434
+ image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
435
# OV2 first-class MM kwarg: per-patch (t,h,w)
436
# positions required by the 3-D vision RoPE.
437
patch_positions=MultiModalFieldConfig.flat_from_sizes(
438
- "image", image_pixel_grid_sizes
438
+ "image", image_pixel_grid_sizes, keep_on_cpu=True
439
),
440
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
441
"video", video_patch_sizes
442
),
443
video_grid_thw=MultiModalFieldConfig.flat_from_sizes(
444
- "video", video_num_frames
444
+ "video", video_num_frames, keep_on_cpu=True
445
),
446
patch_positions_videos=MultiModalFieldConfig.flat_from_sizes(
447
- "video", video_patch_sizes
447
+ "video", video_patch_sizes, keep_on_cpu=True
448
),
449
video_num_frames=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
450
frame_timestamps=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
vllm/model_executor/models/mimo_v2_omni.py
+9
-5
@@ -893,17 +893,21 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
893
merge_size = self.info.get_hf_config().vision_config.spatial_merge_size
894
fields: dict[str, MultiModalFieldConfig] = dict(
895
**_create_qwen2vl_field_factory(merge_size)(hf_inputs),
896
- second_per_grid_ts=MultiModalFieldConfig.batched("video"),
897
- video_start_times=MultiModalFieldConfig.batched("video"),
896
+ second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
897
+ video_start_times=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
898
audio_features=MultiModalFieldConfig.batched("audio"),
899
- audio_token_lens=MultiModalFieldConfig.batched("audio"),
899
+ audio_token_lens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
900
)
901
# video_audio fields: only present when video_audio content was processed
902
if "video_audio_n_segs" in hf_inputs:
903
- fields["video_audio_n_segs"] = MultiModalFieldConfig.batched("video")
903
+ fields["video_audio_n_segs"] = MultiModalFieldConfig.batched(
904
+ "video", keep_on_cpu=True
905
+ )
906
# video_audio_seg_lens: list of per-video 1D tensors, batched("video")
907
if "video_audio_seg_lens" in hf_inputs:
906
- fields["video_audio_seg_lens"] = MultiModalFieldConfig.batched("video")
908
+ fields["video_audio_seg_lens"] = MultiModalFieldConfig.batched(
909
+ "video", keep_on_cpu=True
910
+ )
911
if "va_audio_features" in hf_inputs:
912
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
913
return fields
vllm/model_executor/models/minicpmv.py
+4
-4
@@ -472,12 +472,12 @@ def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
472
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
473
return dict(
474
pixel_values=MultiModalFieldConfig.batched("image"),
475
- image_sizes=MultiModalFieldConfig.batched("image"),
476
- tgt_sizes=MultiModalFieldConfig.batched("image"),
475
+ image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
476
+ tgt_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
477
image_embeds=MultiModalFieldConfig.batched("image"),
478
video_pixel_values=MultiModalFieldConfig.batched("video"),
479
- video_image_sizes=MultiModalFieldConfig.batched("video"),
480
- video_tgt_sizes=MultiModalFieldConfig.batched("video"),
479
+ video_image_sizes=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
480
+ video_tgt_sizes=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
481
video_embeds=MultiModalFieldConfig.batched("video"),
482
)
483
vllm/model_executor/models/minicpmv4_6.py
+6
-4
@@ -75,15 +75,17 @@ from .vision import is_vit_use_data_parallel
75
def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
76
fields = dict(
77
pixel_values=MultiModalFieldConfig.batched("image"),
78
- tgt_sizes=MultiModalFieldConfig.batched("image"),
78
+ tgt_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
79
image_embeds=MultiModalFieldConfig.batched("image"),
80
video_pixel_values=MultiModalFieldConfig.batched("video"),
81
- video_image_sizes=MultiModalFieldConfig.batched("video"),
82
- video_tgt_sizes=MultiModalFieldConfig.batched("video"),
81
+ video_image_sizes=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
82
+ video_tgt_sizes=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
83
video_embeds=MultiModalFieldConfig.batched("video"),
84
)
85
if "use_vit_merger" in hf_inputs:
86
- fields["use_vit_merger"] = MultiModalFieldConfig.batched("image")
86
+ fields["use_vit_merger"] = MultiModalFieldConfig.batched(
87
+ "image", keep_on_cpu=True
88
+ )
89
return fields
90
91
vllm/model_executor/models/mllama4.py
+2
-2
@@ -653,8 +653,8 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
653
pixel_values=MultiModalFieldConfig.flat_from_sizes(
654
"image", patches_per_image
655
),
656
- patches_per_image=MultiModalFieldConfig.batched("image"),
657
- aspect_ratios=MultiModalFieldConfig.batched("image"),
656
+ patches_per_image=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
657
+ aspect_ratios=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
658
)
659
660
def _get_prompt_updates(
vllm/model_executor/models/molmo.py
+4
-2
@@ -1208,8 +1208,10 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1208
images=MultiModalFieldConfig.flat_from_sizes("image", num_crops),
1209
image_masks=MultiModalFieldConfig.flat_from_sizes("image", num_crops),
1210
image_input_idx=MultiModalFieldConfig.flat_from_sizes("image", num_crops),
1211
- num_crops=MultiModalFieldConfig.batched("image"),
1212
- img_patch_id=MultiModalFieldConfig.shared("image", num_images),
1211
+ num_crops=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
1212
+ img_patch_id=MultiModalFieldConfig.shared(
1213
+ "image", num_images, keep_on_cpu=True
1214
+ ),
1215
)
1216
1217
def _get_prompt_updates(
vllm/model_executor/models/molmo2.py
+12
-8
@@ -2127,26 +2127,30 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
2127
image_token_pooling=MultiModalFieldConfig.flat_from_sizes(
2128
"image", image_num_pooled_patches
2129
),
2130
- image_num_crops=MultiModalFieldConfig.batched("image"),
2131
- image_num_pooled_patches=MultiModalFieldConfig.batched("image"),
2132
- image_num_patches=MultiModalFieldConfig.batched("image"),
2130
+ image_num_crops=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
2131
+ image_num_pooled_patches=MultiModalFieldConfig.batched(
2132
+ "image", keep_on_cpu=True
2133
+ ),
2134
+ image_num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
2135
image_tokens=MultiModalFieldConfig.flat_from_sizes(
2136
"image", num_image_tokens
2137
),
2136
- num_image_tokens=MultiModalFieldConfig.batched("image"),
2138
+ num_image_tokens=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
2139
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
2140
"video", video_num_crops
2141
),
2142
video_token_pooling=MultiModalFieldConfig.flat_from_sizes(
2143
"video", video_num_pooled_patches
2144
),
2143
- video_num_crops=MultiModalFieldConfig.batched("video"),
2144
- video_num_pooled_patches=MultiModalFieldConfig.batched("video"),
2145
- video_num_patches=MultiModalFieldConfig.batched("video"),
2145
+ video_num_crops=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
2146
+ video_num_pooled_patches=MultiModalFieldConfig.batched(
2147
+ "video", keep_on_cpu=True
2148
+ ),
2149
+ video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
2150
video_tokens=MultiModalFieldConfig.flat_from_sizes(
2151
"video", num_video_tokens
2152
),
2149
- num_video_tokens=MultiModalFieldConfig.batched("video"),
2153
+ num_video_tokens=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
2154
)
2155
2156
def _get_prompt_updates(
vllm/model_executor/models/moss_transcribe_diarize.py
+5
-2
@@ -352,9 +352,12 @@ def _mtd_field_config(
352
audio_feature_lengths=MultiModalFieldConfig.flat_from_sizes(
353
"audio",
354
audio_chunk_counts,
355
+ keep_on_cpu=True,
356
+ ),
357
+ audio_chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
358
+ audio_token_lengths=MultiModalFieldConfig.batched(
359
+ "audio", keep_on_cpu=True
360
),
356
- audio_chunk_counts=MultiModalFieldConfig.batched("audio"),
357
- audio_token_lengths=MultiModalFieldConfig.batched("audio"),
361
)
362
return fields
363
vllm/model_executor/models/muse_glimmer.py
+6
-2
@@ -378,12 +378,16 @@ class MuseGlimmerMultiModalProcessor(
378
if "image_pixel_values" in hf_inputs:
379
fields.update(
380
image_pixel_values=MultiModalFieldConfig.batched("image"),
381
- image_feature_sizes=MultiModalFieldConfig.batched("image"),
381
+ image_feature_sizes=MultiModalFieldConfig.batched(
382
+ "image", keep_on_cpu=True
383
+ ),
384
)
385
if "video_pixel_values" in hf_inputs:
386
fields.update(
387
video_pixel_values=MultiModalFieldConfig.batched("video"),
386
- video_feature_sizes=MultiModalFieldConfig.batched("video"),
388
+ video_feature_sizes=MultiModalFieldConfig.batched(
389
+ "video", keep_on_cpu=True
390
+ ),
391
)
392
return fields
393
vllm/model_executor/models/nano_nemotron_vl.py
+8
-6
@@ -386,10 +386,12 @@ class NanoNemotronVLMultiModalProcessor(
386
387
return dict(
388
pixel_values_flat=pixel_values_flat,
389
- image_num_patches=MultiModalFieldConfig.batched("image"),
389
+ image_num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
390
image_embeds=MultiModalFieldConfig.batched("image"),
391
- num_tokens_per_image=MultiModalFieldConfig.batched("image"),
392
- imgs_sizes=MultiModalFieldConfig.batched("image"),
391
+ num_tokens_per_image=MultiModalFieldConfig.batched(
392
+ "image", keep_on_cpu=True
393
+ ),
394
+ imgs_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
395
)
396
397
def _get_video_fields_config(self, hf_inputs: BatchFeature):
@@ -399,9 +401,9 @@ class NanoNemotronVLMultiModalProcessor(
401
pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes(
402
"video", video_num_patches
403
),
402
- video_num_patches=MultiModalFieldConfig.batched("video"),
403
- frames_indices=MultiModalFieldConfig.batched("video"),
404
- frame_duration_ms=MultiModalFieldConfig.batched("video"),
404
+ video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
405
+ frames_indices=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
406
+ frame_duration_ms=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
407
)
408
409
def _get_audio_fields_config(self, hf_inputs: BatchFeature):
vllm/model_executor/models/ovis.py
+1
-1
@@ -386,7 +386,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
386
) -> Mapping[str, MultiModalFieldConfig]:
387
return dict(
388
pixel_values=MultiModalFieldConfig.batched("image"),
389
- grids=MultiModalFieldConfig.batched("image"),
389
+ grids=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
390
indicator_tokens=MultiModalFieldConfig.batched("image"),
391
)
392
vllm/model_executor/models/paddleocr_vl.py
+1
-1
@@ -279,7 +279,7 @@ class PaddleOCRVLMultiModalProcessor(
279
) -> Mapping[str, MultiModalFieldConfig]:
280
return dict(
281
pixel_values=MultiModalFieldConfig.batched("image"),
282
- image_grid_thw=MultiModalFieldConfig.batched("image"),
282
+ image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
283
)
284
285
def _get_prompt_updates(
vllm/model_executor/models/phi4mm.py
+1
-1
@@ -918,7 +918,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
918
input_image_embeds=MultiModalFieldConfig.batched("image"),
919
image_attention_mask=MultiModalFieldConfig.batched("image"),
920
image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
921
- num_img_tokens=MultiModalFieldConfig.batched("image"),
921
+ num_img_tokens=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
922
input_audio_embeds=MultiModalFieldConfig.batched("audio"),
923
)
924
vllm/model_executor/models/qwen2_5_omni_thinker.py
+14
-4
@@ -301,8 +301,12 @@ def create_qwen2_5_omni_thinker_field_factory(
301
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
302
"audio", audio_feature_lengths, dim=1
303
),
304
- feature_attention_mask=MultiModalFieldConfig.batched("audio"),
305
- audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
304
+ feature_attention_mask=MultiModalFieldConfig.batched(
305
+ "audio", keep_on_cpu=True
306
+ ),
307
+ audio_feature_lengths=MultiModalFieldConfig.batched(
308
+ "audio", keep_on_cpu=True
309
+ ),
310
pixel_values=MultiModalFieldConfig.flat_from_sizes(
311
"image", image_pixel_grid_sizes
312
),
@@ -318,7 +322,9 @@ def create_qwen2_5_omni_thinker_field_factory(
322
),
323
video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
324
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
321
- use_audio_in_video=MultiModalFieldConfig.shared("video", num_videos),
325
+ use_audio_in_video=MultiModalFieldConfig.shared(
326
+ "video", num_videos, keep_on_cpu=True
327
+ ),
328
)
329
330
return _qwen2_5_omni_thinker_field_config
@@ -1031,7 +1037,11 @@ class Qwen2_5OmniConditionalGenerationMixin:
1037
audio_input: Qwen2_5OmniAudioFeatureInputs,
1038
) -> torch.Tensor:
1039
input_features = audio_input["input_features"]
1034
- audio_feature_lengths = audio_input["audio_feature_lengths"]
1040
+ # audio_feature_lengths is keep_on_cpu; the audio tower derives
1041
+ # device placement from feature_lens, so move it explicitly.
1042
+ audio_feature_lengths = audio_input["audio_feature_lengths"].to(
1043
+ input_features.device, non_blocking=True
1044
+ )
1045
1046
audio_feat_lengths, audio_output_lengths = (
1047
self.audio_tower._get_feat_extract_output_lengths(audio_feature_lengths)
vllm/model_executor/models/qwen2_audio.py
+1
-1
@@ -130,7 +130,7 @@ def _qwen2audio_field_config(hf_inputs: Mapping[str, torch.Tensor]):
130
return dict(
131
audio_embeds=MultiModalFieldConfig.batched("audio"),
132
input_features=MultiModalFieldConfig.batched("audio"),
133
- feature_attention_mask=MultiModalFieldConfig.batched("audio"),
133
+ feature_attention_mask=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
134
)
135
136
vllm/model_executor/models/qwen3_asr.py
+7
-3
@@ -254,8 +254,8 @@ def _qwen3asr_field_config(hf_inputs: Mapping[str, torch.Tensor]):
254
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
255
"audio", audio_feature_lengths, dim=1
256
),
257
- feature_attention_mask=MultiModalFieldConfig.batched("audio"),
258
- audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
257
+ feature_attention_mask=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
258
+ audio_feature_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
259
)
260
261
@@ -459,7 +459,11 @@ class Qwen3ASRForConditionalGeneration(
459
audio_input: Qwen2_5OmniAudioFeatureInputs,
460
) -> torch.Tensor:
461
input_features = audio_input["input_features"]
462
- audio_feature_lengths = audio_input["audio_feature_lengths"]
462
+ # audio_feature_lengths is keep_on_cpu; the audio tower derives
463
+ # device placement from feature_lens, so move it explicitly.
464
+ audio_feature_lengths = audio_input["audio_feature_lengths"].to(
465
+ input_features.device, non_blocking=True
466
+ )
467
468
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
469
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+5
-1
@@ -1553,7 +1553,11 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix
1553
audio_input: Qwen2_5OmniAudioFeatureInputs,
1554
) -> tuple[torch.Tensor, ...]:
1555
input_features = audio_input["input_features"]
1556
- audio_feature_lengths = audio_input["audio_feature_lengths"]
1556
+ # audio_feature_lengths is keep_on_cpu; the audio tower derives
1557
+ # device placement from feature_lens, so move it explicitly.
1558
+ audio_feature_lengths = audio_input["audio_feature_lengths"].to(
1559
+ input_features.device, non_blocking=True
1560
+ )
1561
1562
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
1563
vllm/model_executor/models/step3_vl.py
+1
-1
@@ -202,7 +202,7 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo])
202
),
203
num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
204
patch_newline_mask=MultiModalFieldConfig.flat_from_sizes(
205
- "image", num_patches
205
+ "image", num_patches, keep_on_cpu=True
206
),
207
)
208
vllm/model_executor/models/transformers/multimodal.py
+15
-3
@@ -300,11 +300,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
300
301
# Keep these as batched, as they always have batch size as first dim
302
if "audio" in modalities:
303
- mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
303
+ mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched(
304
+ "audio", keep_on_cpu=True
305
+ )
306
if "image" in modalities:
305
- mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
307
+ mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
308
+ "image", keep_on_cpu=True
309
+ )
310
# TODO: route to "video" once the video modality is supported
307
- mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
311
+ mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
312
+ "image", keep_on_cpu=True
313
+ )
314
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
315
"image", keep_on_cpu=True
316
)
@@ -755,6 +761,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
761
762
num_image_patches = kwargs.pop("num_image_patches")
763
764
+ # grid_thw fields are registered keep_on_cpu; restore the on-device
765
+ # placement that HF get_image_features implementations expect.
766
+ for key, value in kwargs.items():
767
+ if isinstance(value, torch.Tensor):
768
+ kwargs[key] = value.to(pixel_values.device, non_blocking=True)
769
+
770
# The underlying HuggingFace `get_image_features` implementations
771
# contain model-internal syncs (e.g. Idefics3 filters all-zero
772
# padding images via boolean-mask indexing, LlavaOnevision
vllm/model_executor/models/ultravox.py
+1
-1
@@ -251,7 +251,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
251
"audio", num_chunks, keep_on_cpu=True
252
),
253
# num_chunks can convert audio_chunked to audio batch dimension
254
- audio_num_chunks=MultiModalFieldConfig.batched("audio"),
254
+ audio_num_chunks=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
255
audio_embeds=MultiModalFieldConfig.batched("audio"),
256
)
257
vllm/models/inkling/common/mm_preprocess.py
+2
-2
@@ -307,12 +307,12 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
307
return dict(
308
# Ragged per-image patches, grouped by num_patches.
309
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
310
- num_patches=MultiModalFieldConfig.batched("image"),
310
+ num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
311
# Ragged per-audio frames, grouped by num_audio_tokens.
312
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
313
"audio", num_audio_tokens
314
),
315
- num_audio_tokens=MultiModalFieldConfig.batched("audio"),
315
+ num_audio_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
316
)
317
318
def _get_prompt_updates(