A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Use `merge_by_field_config` for MM models (InternVL family) (#26153)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 3, 2025 at 16:59 UTC
f9a8084e48794565c07dd5055ab469eaa4d36160
9 files changed
+84
-182
docs/models/supported_models.md
+1
-1
@@ -677,7 +677,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
677
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | ✅︎ |
678
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ | ✅︎ |
679
| `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | | ✅︎ |
680
-| `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1`, etc. | ✅︎ | ✅︎ | ✅︎ |
680
+| `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1`, `internlm/Intern-S1-mini`, etc. | ✅︎ | ✅︎ | ✅︎ |
681
| `InternVLChatModel` | InternVL 3.5, InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3_5-14B`, `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ | ✅︎ |
682
| `InternVLForConditionalGeneration` | InternVL 3.0 (HF format) | T + I<sup>E+</sup> + V<sup>E+</sup> | `OpenGVLab/InternVL3-1B-hf`, etc. | ✅︎ | ✅︎ | ✅︎ |
683
| `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-8B-Preview` | ✅︎ | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+1
-1
@@ -576,7 +576,7 @@ def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
576
577
# Intern-S1
578
def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
579
- model_name = "internlm/Intern-S1"
579
+ model_name = "internlm/Intern-S1-mini"
580
581
engine_args = EngineArgs(
582
model=model_name,
examples/offline_inference/vision_language_multi_image.py
+1
-1
@@ -309,7 +309,7 @@ def load_idefics3(question: str, image_urls: list[str]) -> ModelRequestData:
309
310
311
def load_interns1(question: str, image_urls: list[str]) -> ModelRequestData:
312
- model_name = "internlm/Intern-S1"
312
+ model_name = "internlm/Intern-S1-mini"
313
314
engine_args = EngineArgs(
315
model=model_name,
vllm/model_executor/models/interns1.py
+19
-53
@@ -25,7 +25,7 @@ from vllm.model_executor.models.interns1_vit import InternS1VisionModel
25
from vllm.model_executor.models.module_mapping import MultiModelKeys
26
from vllm.multimodal import MULTIMODAL_REGISTRY
27
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
28
- MultiModalKwargsItems, NestedTensors)
28
+ MultiModalKwargsItems)
29
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
30
ImageSize, MultiModalDataItems)
31
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -39,7 +39,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
39
40
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
41
SupportsMultiModal, SupportsPP)
42
-from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
42
+from .utils import (AutoWeightsLoader, WeightsMapper,
43
init_vllm_registered_model, maybe_prefix)
44
45
@@ -304,7 +304,7 @@ class InternS1MultiModalProcessor(
304
mm_data: Mapping[str, object],
305
mm_kwargs: Mapping[str, object],
306
tok_kwargs: Mapping[str, object],
307
- ) -> Mapping[str, NestedTensors]:
307
+ ) -> BatchFeature:
308
mm_data = dict(mm_data)
309
videos = mm_data.pop("videos", [])
310
images = mm_data.pop("images", [])
@@ -342,7 +342,7 @@ class InternS1MultiModalProcessor(
342
image_placeholder, 1)
343
344
num_patches = [len(item) for item in image_pixel_values]
345
- image_outputs: dict[str, NestedTensors] = {
345
+ image_outputs = {
346
"pixel_values": torch.concat(image_pixel_values),
347
"image_num_patches": torch.tensor(num_patches),
348
"image_token_id": torch.tensor(hf_processor.image_token_id),
@@ -370,7 +370,7 @@ class InternS1MultiModalProcessor(
370
video_placeholder, 1)
371
372
num_frames = [len(item) for item in video_pixel_values]
373
- video_outputs: dict[str, NestedTensors] = {
373
+ video_outputs = {
374
"pixel_values_videos": torch.concat(video_pixel_values),
375
"video_num_patches": torch.tensor(num_frames),
376
"video_token_id": torch.tensor(video_token_id),
@@ -382,16 +382,11 @@ class InternS1MultiModalProcessor(
382
prompt)
383
text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt")
384
385
- combined_outputs = dict(
386
- **text_outputs,
387
- **image_outputs,
388
- **video_outputs,
389
- )
390
- return BatchFeature(combined_outputs)
385
+ return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
386
387
def _get_mm_fields_config(
388
self,
394
- hf_inputs: Mapping[str, NestedTensors],
389
+ hf_inputs: BatchFeature,
390
hf_processor_mm_kwargs: Mapping[str, object],
391
) -> Mapping[str, MultiModalFieldConfig]:
392
@@ -487,6 +482,7 @@ class InternS1MultiModalProcessor(
482
dummy_inputs=InternS1DummyInputsBuilder)
483
class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
484
SupportsPP, SupportsLoRA):
485
+ merge_by_field_config = True
486
487
# To ensure correct weight loading and mapping.
488
hf_to_vllm_mapper = WeightsMapper(
@@ -561,7 +557,7 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
557
prefix=prefix,
558
)
559
564
- def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
560
+ def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
561
return InternS1MultiModalProjector(config)
562
563
def pixel_shuffle(self, x, scale_factor=0.5):
@@ -599,13 +595,9 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
595
return None
596
597
if image_embeds is not None:
602
- if not isinstance(image_embeds, (torch.Tensor, list)):
603
- raise ValueError("Incorrect type of image embeddings. "
604
- f"Got type: {type(image_embeds)}")
605
-
598
return InternS1ImageEmbeddingInputs(
599
type="image_embeds",
608
- data=flatten_bn(image_embeds),
600
+ data=image_embeds,
601
)
602
603
image_token_id = kwargs["image_token_id"]
@@ -613,17 +605,6 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
605
self.img_context_token_id = image_token_id.flatten().unique().item()
606
607
if pixel_values is not None:
616
- if not isinstance(pixel_values, (torch.Tensor, list)):
617
- raise ValueError("Incorrect type of pixel values. "
618
- f"Got type: {type(pixel_values)}")
619
-
620
- if not isinstance(image_num_patches, (torch.Tensor, list)):
621
- raise ValueError("Incorrect type of image_num_patches. "
622
- f"Got type: {type(image_num_patches)}")
623
-
624
- pixel_values = flatten_bn(pixel_values, concat=True)
625
- image_num_patches = flatten_bn(image_num_patches, concat=True)
626
-
608
h, w = self.config.vision_config.image_size
609
return InternS1ImagePixelInputs(
610
type="pixel_values",
@@ -638,7 +619,7 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
619
raise AssertionError("This line should be unreachable.")
620
621
def _parse_and_validate_video_input(
641
- self, **kwargs: object) -> Optional[InternS1VideoPixelInputs]:
622
+ self, **kwargs: object) -> Optional[InternS1VideoInputs]:
623
pixel_values_flat_video = kwargs.pop("pixel_values_videos", None)
624
video_num_patches = kwargs.pop("video_num_patches", None)
625
video_embeds = kwargs.pop("video_embeds", None)
@@ -647,13 +628,9 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
628
return None
629
630
if video_embeds is not None:
650
- if not isinstance(video_embeds, (torch.Tensor, list)):
651
- raise ValueError("Incorrect type of video embeddings. "
652
- f"Got type: {type(video_embeds)}")
653
-
654
- return InternS1ImageEmbeddingInputs(
631
+ return InternS1VideoEmbeddingInputs(
632
type="video_embeds",
656
- data=flatten_bn(video_embeds),
633
+ data=video_embeds,
634
)
635
636
video_token_id = kwargs["video_token_id"]
@@ -661,18 +638,6 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
638
self.video_context_token_id = video_token_id.flatten().unique().item()
639
640
if pixel_values_flat_video is not None:
664
- if not isinstance(pixel_values_flat_video, (torch.Tensor, list)):
665
- raise ValueError("Incorrect type of pixel values. "
666
- f"Got type: {type(pixel_values_flat_video)}")
667
-
668
- if not isinstance(video_num_patches, (torch.Tensor, list)):
669
- raise ValueError("Incorrect type of image_num_patches. "
670
- f"Got type: {type(video_num_patches)}")
671
-
672
- pixel_values_flat_video = flatten_bn(pixel_values_flat_video,
673
- concat=True)
674
- video_num_patches = flatten_bn(video_num_patches, concat=True)
675
-
641
h, w = self.config.vision_config.image_size
642
return InternS1VideoPixelInputs(
643
type="pixel_values_videos",
@@ -686,11 +651,12 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
651
652
raise AssertionError("This line should be unreachable.")
653
689
- def _process_image_input(
654
+ def _process_vision_input(
655
self,
691
- image_input: Union[InternS1ImageInputs, InternS1VideoPixelInputs],
656
+ image_input: Union[InternS1ImageInputs, InternS1VideoInputs],
657
) -> tuple[torch.Tensor, ...]:
693
- if image_input["type"] == "image_embeds":
658
+ if (image_input["type"] == "image_embeds"
659
+ or image_input["type"] == "video_embeds"):
660
return image_input["data"]
661
662
assert self.vision_tower is not None
@@ -753,11 +719,11 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
719
for modality in modalities:
720
if modality == "images":
721
image_input = modalities["images"]
756
- vision_embeddings = self._process_image_input(image_input)
722
+ vision_embeddings = self._process_vision_input(image_input)
723
multimodal_embeddings += vision_embeddings
724
if modality == "videos":
725
video_input = modalities["videos"]
760
- video_embeddings = self._process_image_input(video_input)
726
+ video_embeddings = self._process_vision_input(video_input)
727
multimodal_embeddings += video_embeddings
728
729
return multimodal_embeddings
vllm/model_executor/models/internvl.py
+27
-54
@@ -17,7 +17,7 @@ import torch
17
import torch.nn as nn
18
import torchvision.transforms as T
19
from PIL import Image
20
-from transformers import BatchEncoding, PretrainedConfig, TensorType
20
+from transformers import BatchFeature, PretrainedConfig, TensorType
21
22
from vllm.config import VllmConfig
23
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -28,7 +28,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
28
from vllm.multimodal import MULTIMODAL_REGISTRY
29
from vllm.multimodal.image import convert_image_mode
30
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31
- MultiModalKwargsItems, NestedTensors)
31
+ MultiModalKwargsItems)
32
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
33
ImageSize, MultiModalDataItems)
34
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -42,8 +42,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
42
43
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
44
SupportsMultiModal, SupportsPP)
45
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
46
- maybe_prefix)
45
+from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
46
47
IMG_START = '<img>'
48
IMG_END = '</img>'
@@ -471,7 +470,7 @@ class BaseInternVLProcessor(ABC):
470
max_dynamic_patch=max_dynamic_patch,
471
dynamic_image_size=dynamic_image_size,
472
)
474
- image_inputs: dict[str, NestedTensors] = {
473
+ image_inputs = {
474
"pixel_values_flat":
475
torch.cat(pixel_values_lst),
476
"image_num_patches":
@@ -502,7 +501,7 @@ class BaseInternVLProcessor(ABC):
501
max_dynamic_patch: Optional[int] = None,
502
dynamic_image_size: Optional[bool] = None,
503
return_tensors: Optional[Union[str, TensorType]] = None,
505
- ) -> Mapping[str, NestedTensors]:
504
+ ) -> BatchFeature:
505
text, images = [self._make_batch_input(x) for x in (text, images)]
506
507
text, image_inputs = self._preprocess_image(
@@ -515,10 +514,9 @@ class BaseInternVLProcessor(ABC):
514
515
text_inputs = self.tokenizer(text)
516
518
- return {
519
- **BatchEncoding(text_inputs, tensor_type=return_tensors),
520
- **image_inputs,
521
- }
517
+ combined_outputs = {**text_inputs, **image_inputs}
518
+
519
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)
520
521
522
class InternVLProcessor(BaseInternVLProcessor):
@@ -598,7 +596,7 @@ class InternVLProcessor(BaseInternVLProcessor):
596
videos,
597
dynamic_image_size=dynamic_image_size,
598
)
601
- video_inputs: dict[str, NestedTensors] = {
599
+ video_inputs = {
600
"pixel_values_flat_video":
601
torch.cat(pixel_values_lst_video),
602
"video_num_patches":
@@ -622,7 +620,7 @@ class InternVLProcessor(BaseInternVLProcessor):
620
max_dynamic_patch: Optional[int] = None,
621
dynamic_image_size: Optional[bool] = None,
622
return_tensors: Optional[Union[str, TensorType]] = None,
625
- ) -> Mapping[str, NestedTensors]:
623
+ ) -> BatchFeature:
624
text, images, videos = [
625
self._make_batch_input(x) for x in (text, images, videos)
626
]
@@ -643,11 +641,9 @@ class InternVLProcessor(BaseInternVLProcessor):
641
642
text_inputs = self.tokenizer(text)
643
646
- return {
647
- **BatchEncoding(text_inputs, tensor_type=return_tensors),
648
- **image_inputs,
649
- **video_inputs,
650
- }
644
+ combined_outputs = {**text_inputs, **image_inputs, **video_inputs}
645
+
646
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)
647
648
def get_image_repl(
649
self,
@@ -773,7 +769,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
769
mm_data: Mapping[str, object],
770
mm_kwargs: Mapping[str, object],
771
tok_kwargs: Mapping[str, object],
776
- ) -> Mapping[str, NestedTensors]:
772
+ ) -> BatchFeature:
773
processed_outputs = super()._call_hf_processor(
774
prompt=prompt,
775
mm_data=mm_data,
@@ -793,7 +789,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
789
790
def _get_mm_fields_config(
791
self,
796
- hf_inputs: Mapping[str, NestedTensors],
792
+ hf_inputs: BatchFeature,
793
hf_processor_mm_kwargs: Mapping[str, object],
794
) -> Mapping[str, MultiModalFieldConfig]:
795
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -948,7 +944,7 @@ class InternVLMultiModalProcessor(
944
mm_data: Mapping[str, object],
945
mm_kwargs: Mapping[str, object],
946
tok_kwargs: Mapping[str, object],
951
- ) -> Mapping[str, NestedTensors]:
947
+ ) -> BatchFeature:
948
processed_outputs = super()._call_hf_processor(prompt, mm_data,
949
mm_kwargs, tok_kwargs)
950
@@ -960,7 +956,7 @@ class InternVLMultiModalProcessor(
956
957
def _get_mm_fields_config(
958
self,
963
- hf_inputs: Mapping[str, NestedTensors],
959
+ hf_inputs: BatchFeature,
960
hf_processor_mm_kwargs: Mapping[str, object],
961
) -> Mapping[str, MultiModalFieldConfig]:
962
image_fields = super()._get_mm_fields_config(hf_inputs,
@@ -1033,6 +1029,7 @@ class InternVLMultiModalProcessor(
1029
dummy_inputs=InternVLDummyInputsBuilder)
1030
class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1031
SupportsLoRA):
1032
+ merge_by_field_config = True
1033
1034
supports_encoder_tp_data = True
1035
@@ -1126,7 +1123,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1123
else:
1124
return InternVisionPatchModel(config.vision_config)
1125
1129
- def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
1126
+ def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
1127
vit_hidden_size = config.vision_config.hidden_size
1128
llm_hidden_size = config.text_config.hidden_size
1129
@@ -1175,13 +1172,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1172
return None
1173
1174
if image_embeds is not None:
1178
- if not isinstance(image_embeds, (torch.Tensor, list)):
1179
- raise ValueError("Incorrect type of image embeddings. "
1180
- f"Got type: {type(image_embeds)}")
1181
-
1175
return InternVLImageEmbeddingInputs(
1176
type="image_embeds",
1184
- data=flatten_bn(image_embeds),
1177
+ data=image_embeds,
1178
)
1179
1180
image_token_id = kwargs["image_token_id"]
@@ -1189,16 +1182,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1182
self.img_context_token_id = image_token_id.flatten().unique().item()
1183
1184
if pixel_values_flat is not None:
1192
- if not isinstance(pixel_values_flat, (torch.Tensor, list)):
1193
- raise ValueError("Incorrect type of pixel values. "
1194
- f"Got type: {type(pixel_values_flat)}")
1195
-
1196
- if not isinstance(image_num_patches, (torch.Tensor, list)):
1197
- raise ValueError("Incorrect type of image_num_patches. "
1198
- f"Got type: {type(image_num_patches)}")
1199
-
1200
- pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
1201
- image_num_patches = flatten_bn(image_num_patches, concat=True)
1185
expected_h = expected_w = self.config.vision_config.image_size
1186
resolve_bindings = {"h": expected_h, "w": expected_w}
1187
@@ -1223,7 +1206,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1206
if video_embeds is not None:
1207
return InternVLVideoEmbeddingInputs(
1208
type="video_embeds",
1226
- data=flatten_bn(video_embeds),
1209
+ data=video_embeds,
1210
)
1211
1212
video_token_id = kwargs["video_token_id"]
@@ -1231,17 +1214,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1214
self.video_context_token_id = video_token_id.flatten().unique().item()
1215
1216
if pixel_values_flat_video is not None:
1234
- if not isinstance(pixel_values_flat_video, (torch.Tensor, list)):
1235
- raise ValueError("Incorrect type of pixel values. "
1236
- f"Got type: {type(pixel_values_flat_video)}")
1237
-
1238
- if not isinstance(video_num_patches, (torch.Tensor, list)):
1239
- raise ValueError("Incorrect type of image_num_patches. "
1240
- f"Got type: {type(video_num_patches)}")
1241
-
1242
- pixel_values_flat_video = flatten_bn(pixel_values_flat_video,
1243
- concat=True)
1244
- video_num_patches = flatten_bn(video_num_patches, concat=True)
1217
expected_h = expected_w = self.config.vision_config.image_size
1218
resolve_bindings = {"h": expected_h, "w": expected_w}
1219
@@ -1254,11 +1226,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1226
1227
raise AssertionError("This line should be unreachable.")
1228
1257
- def _process_image_input(
1229
+ def _process_vision_input(
1230
self,
1259
- image_input: Union[InternVLImageInputs, InternVLVideoPixelInputs],
1231
+ image_input: Union[InternVLImageInputs, InternVLVideoInputs],
1232
) -> tuple[torch.Tensor, ...]:
1261
- if image_input["type"] == "image_embeds":
1233
+ if (image_input["type"] == "image_embeds"
1234
+ or image_input["type"] == "video_embeds"):
1235
return image_input["data"]
1236
1237
assert self.vision_model is not None
@@ -1326,11 +1299,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1299
for modality in modalities:
1300
if modality == "images":
1301
image_input = modalities["images"]
1329
- vision_embeddings = self._process_image_input(image_input)
1302
+ vision_embeddings = self._process_vision_input(image_input)
1303
multimodal_embeddings += vision_embeddings
1304
if modality == "videos":
1305
video_input = modalities["videos"]
1333
- video_embeddings = self._process_image_input(video_input)
1306
+ video_embeddings = self._process_vision_input(video_input)
1307
multimodal_embeddings += video_embeddings
1308
1309
return multimodal_embeddings
vllm/model_executor/models/nano_nemotron_vl.py
+16
-21
@@ -18,8 +18,7 @@ import torch
18
import torch.nn as nn
19
import torchvision.transforms as T
20
from PIL import Image
21
-from transformers import (BatchEncoding, BatchFeature, PretrainedConfig,
22
- TensorType)
21
+from transformers import BatchFeature, PretrainedConfig, TensorType
22
23
from vllm.config import VllmConfig
24
from vllm.model_executor.layers.activation import ReLUSquaredActivation
@@ -38,8 +37,7 @@ from vllm.model_executor.models.utils import (flatten_bn,
37
maybe_prefix)
38
from vllm.multimodal import MULTIMODAL_REGISTRY
39
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
41
- MultiModalKwargs, MultiModalKwargsItems,
42
- NestedTensors)
40
+ MultiModalKwargs, MultiModalKwargsItems)
41
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
42
ImageSize, MultiModalDataItems)
43
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -298,7 +296,7 @@ class BaseNanoNemotronVLProcessor(ABC):
296
else:
297
pixel_values_lst = self._images_to_pixel_values_lst(
298
images, max_num_tiles)
301
- image_inputs: dict[str, NestedTensors] = {
299
+ image_inputs = {
300
"pixel_values_flat":
301
torch.cat(pixel_values_lst),
302
"image_num_patches":
@@ -326,7 +324,7 @@ class BaseNanoNemotronVLProcessor(ABC):
324
images: Optional[Union[Image.Image, list[Image.Image]]] = None,
325
return_tensors: Optional[Union[str, TensorType]] = None,
326
max_num_tiles: Optional[int] = None,
329
- ) -> Mapping[str, NestedTensors]:
327
+ ) -> BatchFeature:
328
# Use default if not provided
329
if max_num_tiles is None:
330
max_num_tiles = 12
@@ -341,10 +339,9 @@ class BaseNanoNemotronVLProcessor(ABC):
339
340
text_inputs = self.tokenizer(text, add_special_tokens=False)
341
344
- return {
345
- **BatchEncoding(text_inputs, tensor_type=return_tensors),
346
- **image_inputs,
347
- }
342
+ combined_outputs = {**text_inputs, **image_inputs}
343
+
344
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)
345
346
347
class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
@@ -420,7 +417,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
417
dynamic_image_size=dynamic_image_size,
418
)
419
423
- video_inputs: dict[str, NestedTensors] = {
420
+ video_inputs = {
421
"pixel_values_flat_video":
422
torch.cat(pixel_values_lst_video),
423
"video_num_patches":
@@ -443,7 +440,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
440
return_tensors: Optional[Union[str, TensorType]] = None,
441
max_num_tiles: Optional[int] = None,
442
dynamic_image_size: Optional[bool] = None,
446
- ) -> Mapping[str, NestedTensors]:
443
+ ) -> BatchFeature:
444
# Use default if not provided
445
if max_num_tiles is None:
446
max_num_tiles = 12
@@ -467,11 +464,9 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
464
465
text_inputs = self.tokenizer(text, add_special_tokens=False)
466
470
- return BatchFeature({
471
- **BatchEncoding(text_inputs, tensor_type=return_tensors),
472
- **image_inputs,
473
- **video_inputs,
474
- })
467
+ combined_outputs = {**text_inputs, **image_inputs, **video_inputs}
468
+
469
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)
470
471
def get_image_repl(
472
self,
@@ -625,7 +620,7 @@ class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
620
mm_data: Mapping[str, object],
621
mm_kwargs: Mapping[str, object],
622
tok_kwargs: Mapping[str, object],
628
- ) -> Mapping[str, NestedTensors]:
623
+ ) -> BatchFeature:
624
processed_outputs = super()._call_hf_processor(
625
prompt=prompt,
626
mm_data=mm_data,
@@ -645,7 +640,7 @@ class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
640
641
def _get_mm_fields_config(
642
self,
648
- hf_inputs: Mapping[str, NestedTensors],
643
+ hf_inputs: BatchFeature,
644
hf_processor_mm_kwargs: Mapping[str, object],
645
) -> Mapping[str, MultiModalFieldConfig]:
646
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -724,7 +719,7 @@ class NanoNemotronVLMultiModalProcessor(
719
mm_data: Mapping[str, object],
720
mm_kwargs: Mapping[str, object],
721
tok_kwargs: Mapping[str, object],
727
- ) -> Mapping[str, NestedTensors]:
722
+ ) -> BatchFeature:
723
processed_outputs = super()._call_hf_processor(prompt, mm_data,
724
mm_kwargs, tok_kwargs)
725
@@ -736,7 +731,7 @@ class NanoNemotronVLMultiModalProcessor(
731
732
def _get_mm_fields_config(
733
self,
739
- hf_inputs: Mapping[str, NestedTensors],
734
+ hf_inputs: BatchFeature,
735
hf_processor_mm_kwargs: Mapping[str, object],
736
) -> Mapping[str, MultiModalFieldConfig]:
737
image_fields = super()._get_mm_fields_config(hf_inputs,
vllm/model_executor/models/nemotron_vl.py
+5
-21
@@ -28,7 +28,6 @@ from vllm.model_executor.models.internvl import (
28
from vllm.model_executor.models.module_mapping import MultiModelKeys
29
from vllm.multimodal import MULTIMODAL_REGISTRY
30
from vllm.multimodal.image import convert_image_mode
31
-from vllm.multimodal.inputs import NestedTensors
31
from vllm.multimodal.processing import PromptUpdateDetails
32
from vllm.sequence import IntermediateTensors
33
from vllm.transformers_utils.processor import (
@@ -37,8 +36,7 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
36
37
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
38
SupportsMultiModal, SupportsPP)
40
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41
- maybe_prefix)
39
+from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
40
41
IMG_START = '<img>'
42
IMG_END = '</img>'
@@ -289,7 +287,7 @@ class NemotronVLProcessor(InternVLProcessor):
287
max_dynamic_patch=max_dynamic_patch,
288
dynamic_image_size=dynamic_image_size,
289
)
292
- image_inputs: dict[str, NestedTensors] = {
290
+ image_inputs = {
291
"pixel_values_flat":
292
torch.cat(pixel_values_lst),
293
"image_num_patches":
@@ -344,6 +342,7 @@ class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
342
dummy_inputs=BaseInternVLDummyInputsBuilder[NemotronVLProcessingInfo])
343
class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
344
SupportsLoRA):
345
+ merge_by_field_config = True
346
347
@classmethod
348
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
@@ -414,7 +413,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
413
return AutoModel.from_config(config.vision_config,
414
trust_remote_code=True)
415
417
- def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
416
+ def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
417
vit_hidden_size = config.vit_hidden_size
418
vision_projection_hidden_size = config.projector_hidden_size
419
llm_hidden_size = config.text_config.hidden_size
@@ -467,13 +466,9 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
466
return None
467
468
if image_embeds is not None:
470
- if not isinstance(image_embeds, (torch.Tensor, list)):
471
- raise ValueError("Incorrect type of image embeddings. "
472
- f"Got type: {type(image_embeds)}")
473
-
469
return InternVLImageEmbeddingInputs(
470
type="image_embeds",
476
- data=flatten_bn(image_embeds),
471
+ data=image_embeds,
472
)
473
474
image_token_id = kwargs["image_token_id"]
@@ -481,17 +476,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
476
self.img_context_token_id = image_token_id.flatten().unique().item()
477
478
if pixel_values_flat is not None:
484
- if not isinstance(pixel_values_flat, (torch.Tensor, list)):
485
- raise ValueError("Incorrect type of pixel values. "
486
- f"Got type: {type(pixel_values_flat)}")
487
-
488
- if not isinstance(image_num_patches, (torch.Tensor, list)):
489
- raise ValueError("Incorrect type of image_num_patches. "
490
- f"Got type: {type(image_num_patches)}")
491
-
492
- pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
493
- image_num_patches = flatten_bn(image_num_patches, concat=True)
494
-
479
return InternVLImagePixelInputs(
480
type="pixel_values",
481
pixel_values_flat=pixel_values_flat,
vllm/model_executor/models/nvlm_d.py
+1
-1
@@ -159,7 +159,7 @@ class NVLMMultiModalProcessor(
159
dummy_inputs=NVLMDummyInputsBuilder)
160
class NVLM_D_Model(InternVLChatModel):
161
162
- def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
162
+ def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
163
vit_hidden_size = config.vision_config.hidden_size
164
llm_intermediate_size = config.text_config.intermediate_size
165
llm_hidden_size = config.text_config.hidden_size
vllm/model_executor/models/skyworkr1v.py
+13
-29
@@ -14,7 +14,7 @@ import torch
14
import torch.nn as nn
15
import torchvision.transforms as T
16
from PIL import Image
17
-from transformers import BatchEncoding, PretrainedConfig, TensorType
17
+from transformers import BatchFeature, PretrainedConfig, TensorType
18
19
from vllm.config import VllmConfig
20
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -25,7 +25,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
25
from vllm.multimodal import MULTIMODAL_REGISTRY
26
from vllm.multimodal.image import convert_image_mode
27
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
28
- MultiModalKwargsItems, NestedTensors)
28
+ MultiModalKwargsItems)
29
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
30
ImageSize, MultiModalDataItems)
31
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -37,8 +37,7 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
37
from vllm.utils.tensor_schema import TensorSchema, TensorShape
38
39
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
40
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41
- maybe_prefix)
40
+from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
41
42
IMG_START = '<img>'
43
IMG_END = '</img>'
@@ -399,7 +398,7 @@ class SkyworkR1VProcessor:
398
max_dynamic_patch: Optional[int] = None,
399
dynamic_image_size: Optional[bool] = None,
400
return_tensors: Optional[Union[str, TensorType]] = None,
402
- ) -> Mapping[str, NestedTensors]:
401
+ ) -> BatchFeature:
402
if text is None:
403
text = []
404
if not isinstance(text, list):
@@ -418,7 +417,7 @@ class SkyworkR1VProcessor:
417
max_dynamic_patch=max_dynamic_patch,
418
dynamic_image_size=dynamic_image_size,
419
)
421
- image_inputs: dict[str, NestedTensors] = {
420
+ image_inputs = {
421
"pixel_values_flat":
422
torch.cat(pixel_values_lst),
423
"image_num_patches":
@@ -435,10 +434,9 @@ class SkyworkR1VProcessor:
434
435
text_inputs = self.tokenizer(text)
436
438
- return {
439
- **BatchEncoding(text_inputs, tensor_type=return_tensors),
440
- **image_inputs,
441
- }
437
+ combined_outputs = {**text_inputs, **image_inputs}
438
+
439
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)
440
441
442
class SkyworkR1VProcessingInfo(BaseProcessingInfo):
@@ -529,7 +527,7 @@ class SkyworkR1VMultiModalProcessor(
527
mm_data: Mapping[str, object],
528
mm_kwargs: Mapping[str, object],
529
tok_kwargs: Mapping[str, object],
532
- ) -> Mapping[str, NestedTensors]:
530
+ ) -> BatchFeature:
531
processed_outputs = super()._call_hf_processor(
532
prompt=prompt,
533
mm_data=mm_data,
@@ -549,7 +547,7 @@ class SkyworkR1VMultiModalProcessor(
547
548
def _get_mm_fields_config(
549
self,
552
- hf_inputs: Mapping[str, NestedTensors],
550
+ hf_inputs: BatchFeature,
551
hf_processor_mm_kwargs: Mapping[str, object],
552
) -> Mapping[str, MultiModalFieldConfig]:
553
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -617,6 +615,7 @@ class SkyworkR1VMultiModalProcessor(
615
info=SkyworkR1VProcessingInfo,
616
dummy_inputs=SkyworkR1VDummyInputsBuilder)
617
class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
618
+ merge_by_field_config = True
619
620
@classmethod
621
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
@@ -703,7 +702,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
702
else:
703
return InternVisionPatchModel(config.vision_config)
704
706
- def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
705
+ def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
706
vit_hidden_size = config.vision_config.hidden_size
707
llm_hidden_size = config.text_config.hidden_size
708
@@ -756,13 +755,9 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
755
return None
756
757
if image_embeds is not None:
759
- if not isinstance(image_embeds, (torch.Tensor, list)):
760
- raise ValueError("Incorrect type of image embeddings. "
761
- f"Got type: {type(image_embeds)}")
762
-
758
return SkyworkR1VImageEmbeddingInputs(
759
type="image_embeds",
765
- data=flatten_bn(image_embeds),
760
+ data=image_embeds,
761
)
762
763
image_token_id = kwargs["image_token_id"]
@@ -770,17 +765,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
765
self.img_context_token_id = image_token_id.flatten().unique().item()
766
767
if pixel_values_flat is not None:
773
- if not isinstance(pixel_values_flat, (torch.Tensor, list)):
774
- raise ValueError("Incorrect type of pixel values. "
775
- f"Got type: {type(pixel_values_flat)}")
776
-
777
- if not isinstance(image_num_patches, (torch.Tensor, list)):
778
- raise ValueError("Incorrect type of image_num_patches. "
779
- f"Got type: {type(image_num_patches)}")
780
-
781
- pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
782
- image_num_patches = flatten_bn(image_num_patches, concat=True)
783
-
768
return SkyworkR1VImagePixelInputs(
769
type="pixel_values",
770
pixel_values_flat=pixel_values_flat,