@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09e974d483

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[Bugfix] Check dimensions of multimodal embeddings in V1 (#15816)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Apr 1, 2025 at 00:01 UTC 09e974d483ab80052fc2586e7d72231c280f46b6

14 files changed +98 -37

examples/offline_inference/vision_language.py

+5 -4

@@ -68,7 +68,7 @@ def run_blip2(questions: list[str], modality: str) -> ModelRequestData:

68 # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa

69 prompts = [f"Question: {question} Answer:" for question in questions]

70 engine_args = EngineArgs(

71 - model="Salesforce/blip2-opt-2.7b",

71 + model="Salesforce/blip2-opt-6.7b",

72 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

73 )

@@ -128,7 +128,8 @@ def run_florence2(questions: list[str], modality: str) -> ModelRequestData:

128 engine_args = EngineArgs(

129 model="microsoft/Florence-2-large",

130 tokenizer="facebook/bart-large",

131 - max_num_seqs=8,

131 + max_model_len=4096,

132 + max_num_seqs=2,

133 trust_remote_code=True,

134 dtype="bfloat16",

135 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

@@ -511,7 +512,7 @@ def run_mllama(questions: list[str], modality: str) -> ModelRequestData:

512 engine_args = EngineArgs(

513 model=model_name,

514 max_model_len=4096,

514 - max_num_seqs=16,

515 + max_num_seqs=2,

516 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

517 )

518

@@ -700,7 +701,7 @@ def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:

701 # NOTE: Need L40 (or equivalent) to avoid OOM

702 engine_args = EngineArgs(

703 model=model_name,

703 - max_model_len=8192,

704 + max_model_len=6144,

705 max_num_seqs=2,

706 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

707 )

tests/distributed/test_pipeline_parallel.py

+1 -1

@@ -217,7 +217,7 @@ EMBEDDING_MODELS = { # type: ignore[var-annotated]

217

218 MULTIMODAL_MODELS = {

219 # [Decoder-only]

220 - "Salesforce/blip2-opt-2.7b": PPTestSettings.fast(),

220 + "Salesforce/blip2-opt-6.7b": PPTestSettings.fast(),

221 "facebook/chameleon-7b": PPTestSettings.fast(),

222 "adept/fuyu-8b": PPTestSettings.fast(),

223 "THUDM/glm-4v-9b": PPTestSettings.fast(),

tests/models/decoder_only/vision_language/test_models.py

+4 -4

@@ -34,8 +34,6 @@ REQUIRES_V0_MODELS = [

34 # V1 Test: no way to fall back for head_dim = 80

35 # https://github.com/vllm-project/vllm/issues/14524

36 "qwen_vl",

37 - "h2ovl",

38 - "blip2",

37 # V1 Test: not enough KV cache space in C1.

38 "fuyu",

39 ]

@@ -161,7 +159,8 @@ VLM_TEST_SETTINGS = {

159 marks=[large_gpu_mark(min_gb=64)],

160 ),

161 "blip2": VLMTestInfo(

164 - models=["Salesforce/blip2-opt-2.7b"],

162 + # TODO: Change back to 2.7b once head_dim = 80 is supported

163 + models=["Salesforce/blip2-opt-6.7b"],

164 test_type=VLMTestType.IMAGE,

165 prompt_formatter=lambda img_prompt: f"Question: {img_prompt} Answer:",

166 img_idx_to_prompt=lambda idx: "",

@@ -248,7 +247,8 @@ VLM_TEST_SETTINGS = {

247 "h2ovl": VLMTestInfo(

248 models = [

249 "h2oai/h2ovl-mississippi-800m",

251 - "h2oai/h2ovl-mississippi-2b",

250 + # TODO: Re-enable once head_dim = 80 is supported

251 + # "h2oai/h2ovl-mississippi-2b",

252 ],

253 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

254 prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501

tests/models/registry.py

+2 -1

@@ -259,7 +259,8 @@ _CROSS_ENCODER_EXAMPLE_MODELS = {

259 _MULTIMODAL_EXAMPLE_MODELS = {

260 # [Decoder-only]

261 "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),

262 - "Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b"), # noqa: E501

262 + "Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b", # noqa: E501

263 + extras={"6b": "Salesforce/blip2-opt-6.7b"}), # noqa: E501

264 "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501

265 "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny", # noqa: E501

266 extras={"fork": "Isotr0py/deepseek-vl2-tiny"}, # noqa: E501

vllm/model_executor/models/florence2.py

+2 -1

@@ -875,7 +875,8 @@ class Florence2MultiModalProcessor(

875 Florence2MultiModalProcessor,

876 info=Florence2ProcessingInfo,

877 dummy_inputs=Florence2DummyInputsBuilder)

878 -class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal):

878 +class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

879 + SupportsV0Only):

880

881 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

882 super().__init__()

vllm/model_executor/models/fuyu.py

+14 -9

@@ -39,7 +39,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

39 PromptUpdate, PromptUpdateDetails)

40 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

41 from vllm.sequence import IntermediateTensors

42 -from vllm.utils import flatten_2d_lists

43 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

44 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

@@ -66,10 +65,13 @@ class FuyuImagePatchInputs(TypedDict):

65 This is used to split the embeddings which has the first two dimensions

66 flattened just like `flat_data`.

67 """

68 +

69 embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

70 """

71 A boolean mask indicating which image embeddings correspond

72 to patch tokens.

73 +

74 + Shape: `(batch_size * num_images, num_embeds)`

75 """

@@ -322,16 +324,18 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

324 def _parse_and_validate_image_input(

325 self, **kwargs: object) -> Optional[FuyuImagePatchInputs]:

326 image_patches = kwargs.pop("image_patches", None)

325 - embed_is_patch = kwargs.pop("embed_is_patch", None)

327 if image_patches is not None:

328 if not isinstance(image_patches, (torch.Tensor, list)):

329 raise ValueError("Incorrect type of image patches. "

330 f"Got type: {type(image_patches)}")

331

332 + embed_is_patch = kwargs.pop("embed_is_patch")

333 if not isinstance(embed_is_patch, (torch.Tensor, list)):

334 raise ValueError("Incorrect type of embed_is_patch. "

335 f"Got type: {type(embed_is_patch)}")

336 +

337 image_patches_flat = flatten_bn(image_patches)

338 + embed_is_patch = flatten_bn(embed_is_patch)

339

340 return FuyuImagePatchInputs(

341 type="image_patches",

@@ -351,6 +355,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

355 assert self.vision_embed_tokens is not None

356 vision_embeddings_flat, _ = self.vision_embed_tokens(

357 image_patches_flat)

358 +

359 return vision_embeddings_flat.split(patches_per_image, dim=0)

360

361 def get_multimodal_embeddings(

@@ -358,13 +363,13 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

363 image_input = self._parse_and_validate_image_input(**kwargs)

364 if image_input is None:

365 return None

361 - vision_embeddings = self._process_image_input(image_input)

362 - #return vision_embeddings

363 - return flatten_2d_lists(

364 - scatter_patch_features(*args) for args in zip(

365 - vision_embeddings,

366 - image_input["embed_is_patch"],

367 - ))

366 +

367 + image_features = self._process_image_input(image_input)

368 +

369 + return scatter_patch_features(

370 + image_features,

371 + image_input["embed_is_patch"],

372 + )

373

374 def get_input_embeddings(

375 self,

vllm/model_executor/models/gemma3_mm.py

+4 -2

@@ -613,7 +613,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

613 def _process_image_input(

614 self,

615 image_input: Gemma3ImageInputs,

616 - ) -> tuple[torch.Tensor, ...]:

616 + ) -> list[torch.Tensor]:

617 assert self.vision_tower is not None

618

619 pixel_values = image_input["pixel_values"]

@@ -625,7 +625,9 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

625 )

626 image_embeds = self.multi_modal_projector(image_features)

627

628 - return image_embeds.split(num_patches.tolist())

628 + return [

629 + e.flatten(0, 1) for e in image_embeds.split(num_patches.tolist())

630 + ]

631

632 def get_multimodal_embeddings(

633 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

vllm/model_executor/models/idefics3.py

+7 -2

@@ -733,7 +733,10 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

733 pixel_attention_mask=pixel_attention_mask,

734 )

735

736 - def _process_image_input(self, image_input: ImageInputs) -> torch.Tensor:

736 + def _process_image_input(

737 + self,

738 + image_input: ImageInputs,

739 + ) -> Union[torch.Tensor, list[torch.Tensor]]:

740 if image_input["type"] == "image_embeds":

741 return image_input["data"]

742

@@ -741,7 +744,9 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

744 image_features = self.model.connector(image_features)

745

746 num_patches = image_input["num_patches"]

744 - return image_features.split(num_patches.tolist())

747 + return [

748 + e.flatten(0, 1) for e in image_features.split(num_patches.tolist())

749 + ]

750

751 def get_multimodal_embeddings(

752 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

vllm/model_executor/models/llava_next_video.py

+5 -4

@@ -406,20 +406,21 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

406 h, w)

407 stacked_embeddings = self._video_pixels_to_features(

408 self.vision_tower, stacked_pixels)

409 - return stacked_embeddings.view(b, num_frames,

410 - *stacked_embeddings.shape[1:])

409 + embeds = stacked_embeddings.view(b, num_frames,

410 + *stacked_embeddings.shape[1:])

411

412 elif is_list_of(video_pixels, torch.Tensor):

413 frames_per_videos = [v.shape[0] for v in video_pixels]

414 stacked_pixels = torch.cat(video_pixels, dim=0)

415 stacked_embeddings = self._video_pixels_to_features(

416 self.vision_tower, stacked_pixels)

417 - return torch.split(stacked_embeddings, frames_per_videos, dim=0)

418 -

417 + embeds = torch.split(stacked_embeddings, frames_per_videos, dim=0)

418 else:

419 raise ValueError(

420 f"Unsupported type of video input {type(video_pixels)}")

421

422 + return [e.flatten(0, 1) for e in embeds]

423 +

424 def get_multimodal_embeddings(

425 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

426 video_input = self._parse_and_validate_video_input(**kwargs)

vllm/model_executor/models/minicpmv.py

+5 -2

@@ -919,8 +919,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

919

920 image_features_flat = self.get_vision_hidden_states(image_input)

921

922 - # Reconstruct the batch dimension

923 - return image_features_flat.split(image_input["num_slices"].tolist())

922 + num_slices = image_input["num_slices"]

923 + return [

924 + e.flatten(0, 1)

925 + for e in image_features_flat.split(num_slices.tolist())

926 + ]

927

928 def _process_multimodal_inputs(self, modalities: dict):

929 # The result multimodal_embeddings is tuple of tensors, with each

vllm/model_executor/models/vision.py

+1 -1

@@ -204,7 +204,7 @@ def scatter_patch_features(

204 (e_is_patch.shape[0], patches_one.shape[-1]),

205 fill_value=torch.nan,

206 )

207 - embed_one[e_is_patch] = patches_one.flatten(0, -2)

207 + embed_one[e_is_patch] = patches_one

208 return embed_one

209

210 return tuple(

vllm/v1/worker/gpu_model_runner.py

+12 -6

@@ -41,6 +41,8 @@ from vllm.v1.utils import bind_kv_cache

41 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch

42 from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin

44 +from .utils import sanity_check_mm_encoder_outputs

45 +

46 if TYPE_CHECKING:

47 import xgrammar as xgr

@@ -867,6 +869,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):

869 curr_group_outputs = self.model.get_multimodal_embeddings(

870 **batched_mm_inputs)

871

872 + sanity_check_mm_encoder_outputs(

873 + curr_group_outputs,

874 + expected_num_items=len(grouped_mm_inputs),

875 + )

876 +

877 for output in curr_group_outputs:

878 encoder_outputs.append(output)

879

@@ -1490,12 +1497,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1497 # Run multimodal encoder.

1498 dummy_encoder_outputs = self.model.get_multimodal_embeddings(

1499 **batched_dummy_mm_inputs)

1493 - assert len(dummy_encoder_outputs) == max_num_mm_items, (

1494 - "Expected dimension 0 of encoder outputs to match the number "

1495 - f"of multimodal data items: {max_num_mm_items}, got "

1496 - f"{len(dummy_encoder_outputs)=} instead. This is most likely "

1497 - "due to the 'get_multimodal_embeddings' method of the model "

1498 - "not implemented correctly.")

1500 +

1501 + sanity_check_mm_encoder_outputs(

1502 + dummy_encoder_outputs,

1503 + expected_num_items=max_num_mm_items,

1504 + )

1505

1506 # Cache the dummy encoder outputs.

1507 self.encoder_cache["tmp"] = dict(enumerate(dummy_encoder_outputs))

vllm/v1/worker/tpu_model_runner.py

@@ -37,6 +37,8 @@ from vllm.v1.sample.tpu.sampler import Sampler as TPUSampler

37 from vllm.v1.utils import bind_kv_cache

38 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch

40 +from .utils import sanity_check_mm_encoder_outputs

41 +

42 if TYPE_CHECKING:

43 from vllm.v1.core.sched.output import SchedulerOutput

@@ -512,6 +514,11 @@ class TPUModelRunner:

514 curr_group_outputs = self.model.get_multimodal_embeddings(

515 **batched_mm_inputs)

516

517 + sanity_check_mm_encoder_outputs(

518 + curr_group_outputs,

519 + expected_num_items=len(grouped_mm_inputs),

520 + )

521 +

522 for output in curr_group_outputs:

523 encoder_outputs.append(output)

524

vllm/v1/worker/utils.py new

+29

@@ -0,0 +1,29 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +import torch

3 +

4 +

5 +def sanity_check_mm_encoder_outputs(

6 + mm_embeddings: object,

7 + expected_num_items: int,

8 +) -> None:

9 + """

10 + Perform sanity checks for the result of

11 + :meth:`vllm.model_executor.models.SupportsMultiModal.get_multimodal_embeddings`.

12 + """

13 + assert isinstance(mm_embeddings, (list, tuple, torch.Tensor)), (

14 + "Expected multimodal embeddings to be a list/tuple of 2D tensors, "

15 + f"or a single 3D tensor, but got {type(mm_embeddings)} "

16 + "instead. This is most likely due to incorrect implementation "

17 + "of the model's `get_multimodal_embeddings` method.")

18 +

19 + assert len(mm_embeddings) == expected_num_items, (

20 + "Expected number of multimodal embeddings to match number of "

21 + f"input items: {expected_num_items}, but got {len(mm_embeddings)=} "

22 + "instead. This is most likely due to incorrect implementation "

23 + "of the model's `get_multimodal_embeddings` method.")

24 +

25 + assert all(e.ndim == 2 for e in mm_embeddings), (

26 + "Expected multimodal embeddings to be a sequence of 2D tensors, "

27 + f"but got tensors with shapes {[e.shape for e in mm_embeddings]} "

28 + "instead. This is most likely due to incorrect implementation "

29 + "of the model's `get_multimodal_embeddings` method.")