ab656f2c2f · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/models/blip2.py

+5 -7

@@ -25,7 +25,7 @@ from vllm.sequence import IntermediateTensors

25

26 from .blip import BlipVisionModel

27 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

28 -from .utils import (AutoWeightsLoader, init_vllm_registered_model,

28 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

29 maybe_prefix, merge_multimodal_embeddings)

30

31 # We use this internally as placeholders since there is no image token

@@ -565,12 +565,11 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

565 return None

566

567 if pixel_values is not None:

568 - if not isinstance(pixel_values, torch.Tensor):

568 + if not isinstance(pixel_values, (torch.Tensor, list)):

569 raise ValueError("Incorrect type of pixel values. "

570 f"Got type: {type(pixel_values)}")

571

572 - # Remove the N dimension until multiple images are supported.

573 - pixel_values = pixel_values.squeeze(1)

572 + pixel_values = flatten_bn(pixel_values, concat=True)

573

574 return Blip2ImagePixelInputs(

575 type="pixel_values",

@@ -578,12 +577,11 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

577 )

578

579 if image_embeds is not None:

581 - if not isinstance(image_embeds, torch.Tensor):

580 + if not isinstance(image_embeds, (torch.Tensor, list)):

581 raise ValueError("Incorrect type of image embeddings. "

582 f"Got type: {type(image_embeds)}")

583

585 - # Remove the N dimension until multiple images are supported.

586 - image_embeds = image_embeds.squeeze(1)

584 + image_embeds = flatten_bn(image_embeds, concat=True)

585

586 return Blip2ImageEmbeddingInputs(

587 type="image_embeds",

vllm/model_executor/models/chameleon.py

+3 -4

@@ -39,7 +39,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

39 from vllm.sequence import IntermediateTensors

40

41 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

42 -from .utils import (is_pp_missing_parameter,

42 +from .utils import (flatten_bn, is_pp_missing_parameter,

43 make_empty_intermediate_tensors_factory, make_layers,

44 maybe_prefix, merge_multimodal_embeddings)

45

@@ -972,12 +972,11 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

972 if pixel_values is None:

973 return None

974

975 - if not isinstance(pixel_values, torch.Tensor):

975 + if not isinstance(pixel_values, (torch.Tensor, list)):

976 raise ValueError("Incorrect type of pixel values. "

977 f"Got type: {type(pixel_values)}")

978

979 - # Remove the N dimension until multiple images are supported.

980 - pixel_values = pixel_values.squeeze(1)

979 + pixel_values = flatten_bn(pixel_values, concat=True)

980

981 return ChameleonImagePixelInputs(

982 type="pixel_values",

vllm/model_executor/models/deepseek_vl2.py

+1 -1

@@ -478,7 +478,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

478 flatten_bn(images_spatial_crop, concat=True)))

479

480 if image_embeds is not None:

481 - if not isinstance(image_embeds, torch.Tensor):

481 + if not isinstance(image_embeds, (torch.Tensor, list)):

482 raise ValueError("Incorrect type of image embeddings. "

483 f"Got type: {type(image_embeds)}")

484

vllm/model_executor/models/glm4v.py

+1 -1

@@ -578,7 +578,7 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

578 pixel_values = kwargs.pop("pixel_values", None)

579

580 if pixel_values is not None:

581 - if not isinstance(pixel_values, torch.Tensor):

581 + if not isinstance(pixel_values, (torch.Tensor, list)):

582 raise ValueError("Incorrect type of pixel values. "

583 f"Got type: {type(pixel_values)}")

584

vllm/model_executor/models/internvl.py

+4 -2

@@ -838,7 +838,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

838 return None

839

840 if image_embeds is not None:

841 - if not isinstance(image_embeds, torch.Tensor):

841 + if not isinstance(image_embeds, (torch.Tensor, list)):

842 raise ValueError("Incorrect type of image embeddings. "

843 f"Got type: {type(image_embeds)}")

844

@@ -856,7 +856,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

856 raise ValueError("Incorrect type of pixel values. "

857 f"Got type: {type(pixel_values_flat)}")

858

859 - assert isinstance(image_num_patches, (torch.Tensor, list))

859 + if not isinstance(image_num_patches, (torch.Tensor, list)):

860 + raise ValueError("Incorrect type of image_num_patches. "

861 + f"Got type: {type(pixel_values_flat)}")

862

863 return InternVLImagePixelInputs(

864 type="pixel_values",

vllm/model_executor/models/llava_next_video.py

+6 -9

@@ -349,21 +349,18 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

349 List[b, Tensor(nb_frames, nb_channels, height, width)]

350 }

351 """

352 - pixel_values = kwargs.pop("pixel_values_videos", None)

352 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)

353

354 - if pixel_values is None:

354 + if pixel_values_videos is None:

355 return None

356

357 - if not (is_list_of(pixel_values,

358 - (torch.Tensor)) # different shape videos

359 - or isinstance(pixel_values,

360 - torch.Tensor)): # same shape videos

361 - raise ValueError("Incorrect type of pixel values. "

362 - f"Got type: {type(pixel_values)}")

357 + if not isinstance(pixel_values_videos, (torch.Tensor, list)):

358 + raise ValueError("Incorrect type of pixel_values_videos. "

359 + f"Got type: {type(pixel_values_videos)}")

360

361 return LlavaNextVideoPixelInputs(

362 type="pixel_values_videos",

366 - data=pixel_values,

363 + data=pixel_values_videos,

364 )

365

366 def _select_image_features(self, image_features: torch.Tensor, *,

vllm/model_executor/models/llava_onevision.py

+1 -4

@@ -574,10 +574,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

574 if pixel_values_videos is None:

575 return None

576

577 - if not (is_list_of(pixel_values_videos,

578 - torch.Tensor) # different shape videos

579 - or isinstance(pixel_values_videos,

580 - torch.Tensor)): # same shape videos

577 + if not isinstance(pixel_values_videos, (torch.Tensor, list)):

578 raise ValueError("Incorrect type of pixel_values_videos. "

579 f"Got type: {type(pixel_values_videos)}")

580

vllm/model_executor/models/paligemma.py

+4 -6

@@ -23,7 +23,7 @@ from vllm.sequence import IntermediateTensors

23

24 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

25 from .siglip import SiglipVisionModel

26 -from .utils import (AutoWeightsLoader, init_vllm_registered_model,

26 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

27 maybe_prefix, merge_multimodal_embeddings)

28 from .vision import get_vision_encoder_info

29

@@ -270,12 +270,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

270 return None

271

272 if pixel_values is not None:

273 - if not isinstance(pixel_values, torch.Tensor):

273 + if not isinstance(pixel_values, (torch.Tensor, list)):

274 raise ValueError("Incorrect type of pixel values. "

275 f"Got type: {type(pixel_values)}")

276

277 - # Remove the N dimension until multiple images are supported.

278 - pixel_values = pixel_values.squeeze(1)

277 + pixel_values = flatten_bn(pixel_values, concat=True)

278

279 return PaliGemmaImagePixelInputs(

280 type="pixel_values",

@@ -287,8 +286,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

286 raise ValueError("Incorrect type of image embeddings. "

287 f"Got type: {type(image_embeds)}")

288

290 - # Remove the N dimension until multiple images are supported.

291 - image_embeds = image_embeds.squeeze(1)

289 + image_embeds = flatten_bn(image_embeds, concat=True)

290

291 return PaliGemmaImageEmbeddingInputs(

292 type="image_embeds",

vllm/model_executor/models/qwen_vl.py

+3 -3

@@ -711,7 +711,7 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

711 image_embeds = kwargs.pop("image_embeds", None)

712

713 if pixel_values is not None:

714 - if not isinstance(pixel_values, torch.Tensor):

714 + if not isinstance(pixel_values, (torch.Tensor, list)):

715 raise ValueError("Incorrect type of pixel values. "

716 f"Got type: {type(pixel_values)}")

717

@@ -722,13 +722,13 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

722 )

723

724 if image_embeds is not None:

725 - if not isinstance(image_embeds, torch.Tensor):

725 + if not isinstance(image_embeds, (torch.Tensor, list)):

726 raise ValueError("Incorrect type of image embeddings. "

727 f"Got type: {type(image_embeds)}")

728

729 return QwenImageEmbeddingInputs(

730 type="image_embeds",

731 - data=flatten_bn(image_embeds),

731 + data=flatten_bn(image_embeds, concat=True),

732 )

733

734 return None

[Bugfix] Loosen type check to avoid errors in V1 (#15021)