@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8693e47e6a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[Bugfix] Fix `mm_hashes` forgetting to be passed (#15668)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 28, 2025 at 13:51 UTC 8693e47e6ab52c1219323141d9b0eba89c4143b7

6 files changed +15 -10

vllm/inputs/preprocess.py

@@ -528,6 +528,7 @@ class InputPreprocessor:

528 prompt_token_ids=decoder_inputs_to_override[

529 "prompt_token_ids"],

530 mm_kwargs=inputs["mm_kwargs"],

531 + mm_hashes=inputs["mm_hashes"],

532 mm_placeholders=inputs["mm_placeholders"],

533 )

534 else:

@@ -536,6 +537,7 @@ class InputPreprocessor:

537 prompt=inputs["prompt"],

538 prompt_token_ids=inputs["prompt_token_ids"],

539 mm_kwargs=inputs["mm_kwargs"],

540 + mm_hashes=inputs["mm_hashes"],

541 mm_placeholders=inputs["mm_placeholders"],

542 )

543 elif inputs["type"] == "token":

vllm/model_executor/models/llava.py

@@ -868,6 +868,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

868 mm_items = self._to_mm_items(mm_data)

869 mm_item_counts = mm_items.get_all_counts()

870 mm_kwargs = result["mm_kwargs"]

871 + mm_hashes = result["mm_hashes"]

872

873 # We reimplement the functionality of MLlavaProcessor from

874 # https://github.com/TIGER-AI-Lab/Mantis.git

@@ -916,6 +917,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

917 prompt=prompt,

918 prompt_token_ids=prompt_ids,

919 mm_kwargs=mm_kwargs,

920 + mm_hashes=mm_hashes,

921 mm_placeholders=mm_placeholder_ranges,

922 )

923

vllm/model_executor/models/mllama.py

+1 -1

@@ -1378,7 +1378,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,

1378 # Because attn_metadata.encoder_seq_lens only counts the last

1379 # group of images for each sample, which is used to cheat the

1380 # block manager to allocate blocks for those images only.

1381 - # See input_processor_for_mllama() for more details.

1381 + # See MllamaMultiModalProcessor for more details.

1382 num_tiles_tensor = kwargs.pop("num_tiles")

1383 num_tiles = [t.tolist() for t in num_tiles_tensor]

1384 num_tokens_per_tile = calc_token_per_chunk(self.image_size)

vllm/model_executor/models/phi4mm.py

+8 -8

@@ -28,7 +28,7 @@ from vllm.model_executor.models.llama import LlamaModel

28 from vllm.model_executor.models.module_mapping import MultiModelKeys

29 from vllm.model_executor.sampling_metadata import SamplingMetadata

30 from vllm.multimodal import MULTIMODAL_REGISTRY

31 -from vllm.multimodal.inputs import MultiModalInputs, NestedTensors

31 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors

32 from vllm.sequence import IntermediateTensors, SequenceData

33 from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

@@ -1319,9 +1319,9 @@ def dummy_data_for_phi4mm(ctx: InputContext, seq_len: int,

1319

1320

1321 def input_mapper_for_phi4mm_audio(ctx: InputContext,

1322 - data: object) -> MultiModalInputs:

1322 + data: object) -> MultiModalKwargs:

1323 """

1324 - This function is used to create the MultiModalInputs for the Phi4MM

1324 + This function is used to create the MultiModalKwargs for the Phi4MM

1325 (audio) model.

1326 Specifically, for audio, we extract the audio features from the sound

1327 file and create pairs of audio features and audio embed lengths (the

@@ -1338,13 +1338,13 @@ def input_mapper_for_phi4mm_audio(ctx: InputContext,

1338 data (object): Audio data.

1339

1340 Returns:

1341 - MultiModalInputs: Multi-modal inputs.

1341 + MultiModalKwargs: Multi-modal inputs.

1342 """

1343 if not isinstance(data, list):

1344 data = [data]

1345

1346 if len(data) == 0:

1347 - return MultiModalInputs()

1347 + return MultiModalKwargs()

1348

1349 audio_features = []

1350 for audio_input in data:

@@ -1365,7 +1365,7 @@ def input_mapper_for_phi4mm_audio(ctx: InputContext,

1365 [single_audio_embed_size],

1366 )

1367 audio_features.append(single_audio_feature_audio_len_pair)

1368 - return MultiModalInputs({"audio_features": audio_features})

1368 + return MultiModalKwargs({"audio_features": audio_features})

1369

1370

1371 def input_mapper_for_phi4mm_image(ctx: InputContext, data: object):

@@ -1373,7 +1373,7 @@ def input_mapper_for_phi4mm_image(ctx: InputContext, data: object):

1373 data = [data]

1374 # data: list of PIL images

1375 if len(data) == 0:

1376 - return MultiModalInputs()

1376 + return MultiModalKwargs()

1377 hf_config = ctx.get_hf_config()

1378 vision_encoder_name = hf_config.img_processor

1379 if vision_encoder_name is None:

@@ -1385,7 +1385,7 @@ def input_mapper_for_phi4mm_image(ctx: InputContext, data: object):

1385

1386 image_input_dict = preprocess(data, dynamic_hd_size, vit_image_size,

1387 vit_patch_size)

1388 - return MultiModalInputs({

1388 + return MultiModalKwargs({

1389 "pixel_values":

1390 image_input_dict["pixel_values"],

1391 "image_sizes":

vllm/model_executor/models/prithvi_geospatial_mae.py

@@ -105,6 +105,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):

105 prompt=prompt,

106 prompt_token_ids=[1],

107 mm_kwargs=MultiModalKwargs(mm_kwargs),

108 + mm_hashes=None,

109 mm_placeholders={},

110 )

111

vllm/multimodal/inputs.py

+1 -1

@@ -743,7 +743,7 @@ class MultiModalInputs(TypedDict):

743 mm_kwargs: MultiModalKwargs

744 """Keyword arguments to be directly passed to the model after batching."""

745

746 - mm_hashes: NotRequired[Optional["MultiModalHashDict"]]

746 + mm_hashes: Optional["MultiModalHashDict"]

747 """The hashes of the multi-modal data."""

748

749 mm_placeholders: MultiModalPlaceholderDict