@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ffa443afed

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[Bugfix] Fix embedding assignment for InternVL-based models (#15086)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 20, 2025 at 11:40 UTC ffa443afedd3ffefb8dbe20607692950b78c1496

7 files changed +123 -106

examples/offline_inference/vision_language.py

-1

@@ -169,7 +169,6 @@ def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:

169 model=model_name,

170 max_model_len=2048,

171 max_num_seqs=2,

172 - # Default is False; setting it to True is not supported in V1 yet

172 mm_processor_kwargs={"do_pan_and_scan": True},

173 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

174 )

examples/offline_inference/vision_language_multi_image.py

-2

@@ -91,8 +91,6 @@ def load_gemma3(question: str, image_urls: list[str]) -> ModelRequestData:

91 model=model_name,

92 max_model_len=8192,

93 max_num_seqs=2,

94 - # Default is False; setting it to True is not supported in V1 yet

95 - mm_processor_kwargs={"do_pan_and_scan": True},

94 limit_mm_per_prompt={"image": len(image_urls)},

95 )

vllm/model_executor/models/gemma3_mm.py

+1 -1

@@ -183,7 +183,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

183 image_width: int,

184 image_height: int,

185 processor: Optional[Gemma3Processor],

186 - ) -> PromptUpdateDetails:

186 + ) -> PromptUpdateDetails[str]:

187 if processor is None:

188 processor = self.get_hf_processor()

189

vllm/model_executor/models/h2ovl.py

+6 -16

@@ -249,20 +249,15 @@ class H2OVLProcessor(BaseInternVLProcessor):

249 def image_token_id(self) -> int:

250 return self.tokenizer.get_vocab()[IMG_CONTEXT]

251

252 - def get_image_repl_features(

252 + def get_image_repl(

253 self,

254 feature_size: int,

255 num_patches: Optional[int],

256 - ) -> str:

257 - return IMG_CONTEXT * feature_size

256 + ) -> PromptUpdateDetails[str]:

257 + repl_features = IMG_CONTEXT * feature_size

258 + repl_full = IMG_START + repl_features + IMG_END

259

259 - def get_image_repl_full(

260 - self,

261 - feature_size: int,

262 - num_patches: Optional[int],

263 - ) -> str:

264 - features = self.get_image_repl_features(feature_size, num_patches)

265 - return IMG_START + features + IMG_END

260 + return PromptUpdateDetails(full=repl_full, features=repl_features)

261

262 def resolve_min_max_num(

263 self,

@@ -501,12 +496,7 @@ class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]

496 if num_patches is not None:

497 assert isinstance(num_patches, int)

498

504 - return PromptUpdateDetails(

505 - full=hf_processor.get_image_repl_full(feature_size,

506 - num_patches),

507 - features=hf_processor.get_image_repl_features(

508 - feature_size, num_patches),

509 - )

499 + return hf_processor.get_image_repl(feature_size, num_patches)

500

501 return [

502 PromptReplacement(

vllm/model_executor/models/internvl.py

+104 -69

@@ -9,14 +9,13 @@

9 from abc import ABC, abstractmethod

10 from collections.abc import Iterable, Mapping, Sequence

11 from functools import cached_property

12 -from typing import (List, Literal, Optional, Set, Tuple, TypedDict, TypeVar,

13 - Union)

12 +from typing import Literal, Optional, Set, Tuple, TypedDict, TypeVar, Union

14 import torch

15 import torch.nn as nn

16 import torchvision.transforms as T

17 from PIL import Image

19 -from transformers import BatchFeature, PretrainedConfig, TensorType

18 +from transformers import BatchEncoding, PretrainedConfig, TensorType

20 from vllm.config import VllmConfig

21 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -36,10 +35,12 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

35 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

36 from vllm.sequence import IntermediateTensors

37 from vllm.transformers_utils.tokenizer import AnyTokenizer

38 +from vllm.utils import flatten_2d_lists

40 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

41 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

42 maybe_prefix, merge_multimodal_embeddings)

43 +from .vision import scatter_patch_features, select_patch_features

45 IMG_START = '<img>'

46 IMG_END = '</img>'

@@ -51,16 +52,26 @@ IMAGENET_STD = (0.229, 0.224, 0.225)

53 class InternVLImagePixelInputs(TypedDict):

54 type: Literal["pixel_values"]

54 - data: torch.Tensor

55 + pixel_values_flat: torch.Tensor

56 """

57 Shape:

58 `(batch_size * num_images * (1 + num_patches), num_channels, height, width)`

59 """

59 - patches_per_image: List[int]

60 +

61 + num_patches: torch.Tensor

62 + """Shape: `(batch_size * num_images)`"""

63 +

64 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

65 """

61 - List of number of total patches for each image in the batch.

66 + A boolean mask indicating which image embeddings correspond

67 + to patch tokens.

68 +

69 + Shape: `(batch_size, num_images, num_embeds)`

70 """

72 + num_embeds: Union[torch.Tensor, list[torch.Tensor]]

73 + """Shape: `(batch_size, num_images)`"""

74 +

76 class InternVLImageEmbeddingInputs(TypedDict):

77 type: Literal["image_embeds"]

@@ -286,19 +297,11 @@ class BaseInternVLProcessor(ABC):

297 raise NotImplementedError

298

299 @abstractmethod

289 - def get_image_repl_features(

300 + def get_image_repl(

301 self,

302 feature_size: int,

303 num_patches: Optional[int],

293 - ) -> str:

294 - raise NotImplementedError

295 -

296 - @abstractmethod

297 - def get_image_repl_full(

298 - self,

299 - feature_size: int,

300 - num_patches: Optional[int],

301 - ) -> str:

304 + ) -> PromptUpdateDetails[str]:

305 raise NotImplementedError

306

307 def resolve_min_max_num(

@@ -394,7 +397,7 @@ class BaseInternVLProcessor(ABC):

397 max_dynamic_patch: Optional[int] = None,

398 dynamic_image_size: Optional[bool] = None,

399 return_tensors: Optional[Union[str, TensorType]] = None,

397 - ) -> BatchFeature:

400 + ) -> Mapping[str, NestedTensors]:

401 if text is None:

402 text = []

403 if not isinstance(text, list):

@@ -413,28 +416,41 @@ class BaseInternVLProcessor(ABC):

416 max_dynamic_patch=max_dynamic_patch,

417 dynamic_image_size=dynamic_image_size,

418 )

416 - image_inputs = {

417 - "pixel_values_flat": torch.cat(pixel_values_lst),

418 - "image_num_patches": list(map(len, pixel_values_lst)),

419 + image_inputs: dict[str, NestedTensors] = {

420 + "pixel_values_flat":

421 + torch.cat(pixel_values_lst),

422 + "image_num_patches":

423 + torch.tensor([len(item) for item in pixel_values_lst]),

424 }

425

426 + tokenizer = self.tokenizer

427 + image_token_id = self.image_token_id

428 +

429 + num_embeds = list[int]()

430 + embed_is_patch = list[torch.Tensor]()

431 +

432 for pixel_values in pixel_values_lst:

433 num_patches = pixel_values.shape[0]

434 feature_size = num_patches * self.num_image_token

435

425 - image_repl = self.get_image_repl_full(feature_size,

426 - num_patches)

427 - text = [t.replace('<image>', image_repl, 1) for t in text]

436 + image_repl = self.get_image_repl(feature_size, num_patches)

437 + feature_tokens = tokenizer.encode(image_repl.features,

438 + add_special_tokens=False)

439 +

440 + text = [t.replace('<image>', image_repl.full, 1) for t in text]

441 + num_embeds.append(len(feature_tokens))

442 + embed_is_patch.append(

443 + torch.tensor(feature_tokens) == image_token_id)

444 +

445 + image_inputs["num_embeds"] = torch.tensor(num_embeds)

446 + image_inputs["embed_is_patch"] = embed_is_patch

447

448 text_inputs = self.tokenizer(text)

449

431 - return BatchFeature(

432 - {

433 - **text_inputs,

434 - **image_inputs,

435 - },

436 - tensor_type=return_tensors,

437 - )

450 + return {

451 + **BatchEncoding(text_inputs, tensor_type=return_tensors),

452 + **image_inputs,

453 + }

454

455

456 class InternVLProcessor(BaseInternVLProcessor):

@@ -443,20 +459,15 @@ class InternVLProcessor(BaseInternVLProcessor):

459 def image_token_id(self) -> int:

460 return self.tokenizer.get_vocab()[IMG_CONTEXT]

461

446 - def get_image_repl_features(

462 + def get_image_repl(

463 self,

464 feature_size: int,

465 num_patches: Optional[int],

450 - ) -> str:

451 - return IMG_CONTEXT * feature_size

466 + ) -> PromptUpdateDetails[str]:

467 + repl_features = IMG_CONTEXT * feature_size

468 + repl_full = IMG_START + repl_features + IMG_END

469

453 - def get_image_repl_full(

454 - self,

455 - feature_size: int,

456 - num_patches: Optional[int],

457 - ) -> str:

458 - features = self.get_image_repl_features(feature_size, num_patches)

459 - return IMG_START + features + IMG_END

470 + return PromptUpdateDetails(full=repl_full, features=repl_features)

471

472

473 class BaseInternVLProcessingInfo(BaseProcessingInfo):

@@ -566,16 +577,15 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

577 prompt: str,

578 mm_data: Mapping[str, object],

579 mm_kwargs: Mapping[str, object],

569 - ) -> BatchFeature:

580 + ) -> Mapping[str, NestedTensors]:

581 processed_outputs = super()._call_hf_processor(

582 prompt=prompt,

583 mm_data=mm_data,

584 mm_kwargs=mm_kwargs,

585 )

586

576 - image_token_id = self.info.get_hf_processor(**mm_kwargs).image_token_id

577 - image_data = mm_data.get("images", [])

578 - assert isinstance(image_data, list)

587 + hf_processor = self.info.get_hf_processor(**mm_kwargs)

588 + image_token_id = hf_processor.image_token_id

589

590 # Since there may be extra tokens in the feature placeholders,

591 # we need to pass the image token ID to the model to select the

@@ -586,7 +596,7 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

596

597 def _get_mm_fields_config(

598 self,

589 - hf_inputs: BatchFeature,

599 + hf_inputs: Mapping[str, NestedTensors],

600 hf_processor_mm_kwargs: Mapping[str, object],

601 ) -> Mapping[str, MultiModalFieldConfig]:

602 image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))

@@ -596,6 +606,8 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

606 pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(

607 "image", image_num_patches),

608 image_num_patches=MultiModalFieldConfig.batched("image"),

609 + embed_is_patch=MultiModalFieldConfig.batched("image"),

610 + num_embeds=MultiModalFieldConfig.batched("image"),

611 image_embeds=MultiModalFieldConfig.batched("image"),

612 image_token_id=MultiModalFieldConfig.shared("image", num_images),

613 )

@@ -637,12 +649,7 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

649 if num_patches is not None:

650 assert isinstance(num_patches, int)

651

640 - return PromptUpdateDetails(

641 - full=hf_processor.get_image_repl_full(feature_size,

642 - num_patches),

643 - features=hf_processor.get_image_repl_features(

644 - feature_size, num_patches),

645 - )

652 + return hf_processor.get_image_repl(feature_size, num_patches)

653

654 return [

655 PromptReplacement(

@@ -832,6 +839,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

839 self, **kwargs: object) -> Optional[InternVLImageInputs]:

840 pixel_values_flat = kwargs.pop("pixel_values_flat", None)

841 image_num_patches = kwargs.pop("image_num_patches", None)

842 + embed_is_patch = kwargs.pop("embed_is_patch", None)

843 + num_embeds = kwargs.pop("num_embeds", None)

844 image_embeds = kwargs.pop("image_embeds", None)

845

846 if pixel_values_flat is None and image_embeds is None:

@@ -858,35 +867,47 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

867

868 if not isinstance(image_num_patches, (torch.Tensor, list)):

869 raise ValueError("Incorrect type of image_num_patches. "

861 - f"Got type: {type(pixel_values_flat)}")

870 + f"Got type: {type(image_num_patches)}")

871 +

872 + if not isinstance(embed_is_patch, (torch.Tensor, list)):

873 + raise ValueError("Incorrect type of embed_is_patch. "

874 + f"Got type: {type(embed_is_patch)}")

875 +

876 + if not isinstance(num_embeds, (torch.Tensor, list)):

877 + raise ValueError("Incorrect type of num_embeds. "

878 + f"Got type: {type(num_embeds)}")

879 +

880 + pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)

881 + image_num_patches = flatten_bn(image_num_patches, concat=True)

882

883 return InternVLImagePixelInputs(

884 type="pixel_values",

865 - data=self._validate_pixel_values(

866 - flatten_bn(pixel_values_flat, concat=True)),

867 - patches_per_image=flatten_bn(image_num_patches,

868 - concat=True).tolist())

885 + pixel_values_flat=self._validate_pixel_values(

886 + pixel_values_flat),

887 + num_patches=image_num_patches,

888 + embed_is_patch=embed_is_patch,

889 + num_embeds=num_embeds,

890 + )

891

892 raise AssertionError("This line should be unreachable.")

893

894 def _process_image_input(

895 self,

896 image_input: InternVLImageInputs,

875 - ) -> tuple[torch.Tensor, ...]:

897 + ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:

898 if image_input["type"] == "image_embeds":

899 return image_input["data"]

900

901 assert self.vision_model is not None

902

881 - image_embeds = self.extract_feature(image_input["data"])

903 + image_embeds = self.extract_feature(image_input["pixel_values_flat"])

904

883 - patches_per_image = image_input["patches_per_image"]

905 + num_patches = image_input["num_patches"]

906

907 # Only one image in the current batch

886 - if len(patches_per_image) == 1:

887 - image_embeds = image_embeds.view(

908 + if len(num_patches) == 1:

909 + return image_embeds.view(

910 -1, self.config.text_config.hidden_size).unsqueeze(0)

889 - return image_embeds

911

912 # NOTE: Image embeddings are split into separate tensors for each image

913 # by the size of each embedding.

@@ -894,10 +915,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

915 image_embeds = image_embeds.view(-1,

916 self.config.text_config.hidden_size)

917 image_feature_sizes = [

897 - num_patches * feature_size for num_patches in patches_per_image

918 + num_patches * feature_size for num_patches in num_patches

919 ]

899 - image_embeds = image_embeds.split(image_feature_sizes)

900 - return image_embeds

920 + return image_embeds.split(image_feature_sizes)

921

922 def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:

923 if self.is_mono:

@@ -911,8 +931,19 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

931 image_input = self._parse_and_validate_image_input(**kwargs)

932 if image_input is None:

933 return None

914 - vision_embeddings = self._process_image_input(image_input)

915 - return vision_embeddings

934 +

935 + image_features = self._process_image_input(image_input)

936 +

937 + if (kwargs.get("v0_path", False)

938 + or image_input["type"] != "pixel_values"):

939 + return image_features

940 +

941 + return flatten_2d_lists(

942 + scatter_patch_features(*args) for args in zip(

943 + image_features,

944 + image_input["num_embeds"],

945 + image_input["embed_is_patch"],

946 + ))

947

948 def get_input_embeddings(

949 self,

@@ -924,8 +955,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

955 assert self.img_context_token_id is not None

956 self._set_visual_token_mask(input_ids)

957 inputs_embeds = merge_multimodal_embeddings(

927 - input_ids, inputs_embeds, multimodal_embeddings,

928 - self.img_context_token_id)

958 + input_ids,

959 + inputs_embeds,

960 + select_patch_features(multimodal_embeddings),

961 + self.img_context_token_id,

962 + )

963 return inputs_embeds

964

965 def forward(

@@ -944,6 +978,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

978 # NOTE: In v1, inputs_embeds is always generated at model runner, this

979 # condition is for v0 compatibility.

980 elif inputs_embeds is None:

981 + kwargs.update({"v0_path": True})

982 vision_embeddings = self.get_multimodal_embeddings(**kwargs)

983 inputs_embeds = self.get_input_embeddings(input_ids,

984 vision_embeddings)

vllm/model_executor/models/nvlm_d.py

+8 -13

@@ -36,11 +36,11 @@ class NVLMProcessor(BaseInternVLProcessor):

36 def image_token_id(self) -> int:

37 return self.tokenizer.get_vocab()[IMG_PAD]

39 - def get_image_repl_features(

39 + def get_image_repl(

40 self,

41 feature_size: int,

42 num_patches: Optional[int],

43 - ) -> str:

43 + ) -> PromptUpdateDetails[str]:

44 if num_patches is None:

45 raise NotImplementedError("Embedding inputs are not supported")

@@ -55,14 +55,9 @@ class NVLMProcessor(BaseInternVLProcessor):

55 # We include the start and end as well because "<Image><tile" is

56 # tokenized as ["<Image", "><", "tile"], resulting in assertion error

57 # when trying to find "<tile" as a subsequence of "<Image><tile"

58 - return "<Image>" + features + "</Image>"

58 + repl = "<Image>" + features + "</Image>"

60 - def get_image_repl_full(

61 - self,

62 - feature_size: int,

63 - num_patches: Optional[int],

64 - ) -> str:

65 - return self.get_image_repl_features(feature_size, num_patches)

60 + return PromptUpdateDetails(full=repl, features=repl)

63 class NVLMProcessingInfo(BaseInternVLProcessingInfo):

@@ -180,11 +175,11 @@ class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):

175 if num_patches is not None:

176 assert isinstance(num_patches, int)

177

178 + repl = hf_processor.get_image_repl(feature_size, num_patches)

179 +

180 return PromptUpdateDetails(

184 - full=hf_processor.get_image_repl_full(feature_size,

185 - num_patches) + "\n",

186 - features=hf_processor.get_image_repl_features(

187 - feature_size, num_patches) + "\n",

181 + full=repl.full + "\n",

182 + features=repl.features + "\n",

183 )

184

185 # See note in dummy data regarding why we have the extra newline

vllm/multimodal/processing.py

+4 -4

@@ -103,13 +103,13 @@ The token sequence or text to update.

103

104

105 @dataclass

106 -class PromptUpdateDetails:

106 +class PromptUpdateDetails(Generic[_S]):

107 """Details about the token sequence or text that are part of the update."""

108

109 - full: PromptSeq

109 + full: _S

110 """The full content."""

111

112 - features: PromptSeq

112 + features: _S

113 """

114 The part of the content that corresponds to feature placeholders;

115 this will be replaced by the output of the vision encoder during model

@@ -117,7 +117,7 @@ class PromptUpdateDetails:

117 """

118

119 @staticmethod

120 - def from_seq(seq: PromptSeq) -> "PromptUpdateDetails":

120 + def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":

121 return PromptUpdateDetails(full=seq, features=seq)

122

123