@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec79b67c77

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[Misc][V1] Avoid using `envs.VLLM_USE_V1` in mm processing (#14256)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Mar 4, 2025 at 23:37 UTC ec79b67c77d57d350709a9ea35ebc9314481994a

7 files changed +38 -8

vllm/inputs/preprocess.py

+22 -2

@@ -254,6 +254,7 @@ class InputPreprocessor:

254 mm_data: MultiModalDataDict,

255 mm_processor_kwargs: Optional[Mapping[str, object]],

256 lora_request: Optional[LoRARequest],

257 + return_mm_hashes: bool = False,

258 ) -> MultiModalInputs:

259 """

260 Apply the model's multi-modal processor to a multi-modal prompt,

@@ -274,7 +275,8 @@ class InputPreprocessor:

275 if mm_processor_kwargs is None:

276 mm_processor_kwargs = {}

277

277 - return mm_processor.apply(prompt, mm_data, mm_processor_kwargs)

278 + return mm_processor.apply(prompt, mm_data, mm_processor_kwargs,

279 + return_mm_hashes)

280

281 async def _process_multimodal_async(

282 self,

@@ -282,6 +284,7 @@ class InputPreprocessor:

284 mm_data: MultiModalDataDict,

285 mm_processor_kwargs: Optional[Mapping[str, object]],

286 lora_request: Optional[LoRARequest],

287 + return_mm_hashes: bool = False,

288 ) -> MultiModalInputs:

289 """Async version of :meth:`_process_multimodal`."""

290 # At the moment on model (PrithviGeoSpatialMAE) requires to be

@@ -299,13 +302,15 @@ class InputPreprocessor:

302 if mm_processor_kwargs is None:

303 mm_processor_kwargs = {}

304

302 - return mm_processor.apply(prompt, mm_data, mm_processor_kwargs)

305 + return mm_processor.apply(prompt, mm_data, mm_processor_kwargs,

306 + return_mm_hashes)

307

308 def _prompt_to_llm_inputs(

309 self,

310 prompt: SingletonPrompt,

311 request_id: str,

312 lora_request: Optional[LoRARequest] = None,

313 + return_mm_hashes: bool = False,

314 ) -> SingletonInputs:

315 """

316 Extract the singleton inputs from a prompt.

@@ -315,6 +320,7 @@ class InputPreprocessor:

320 * request_id

321 * prompt: single encoder or decoder input prompt

322 * lora_request: this is only valid for decoder prompts

323 + * return_mm_hashes: whether to return multimodal hashes

324

325 Returns:

326

@@ -349,6 +355,7 @@ class InputPreprocessor:

355 multi_modal_data,

356 mm_processor_kwargs,

357 lora_request=lora_request,

358 + return_mm_hashes=return_mm_hashes,

359 )

360

361 return token_inputs(

@@ -695,6 +702,7 @@ class InputPreprocessor:

702 request_id: str,

703 lora_request: Optional[LoRARequest] = None,

704 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

705 + return_mm_hashes: bool = False,

706 ) -> DecoderOnlyInputs:

707 """

708 For decoder-only models:

@@ -706,6 +714,7 @@ class InputPreprocessor:

714 * request_id

715 * lora_request

716 * prompt_adapter_request

717 + * return_mm_hashes

718

719 Returns:

720

@@ -729,6 +738,7 @@ class InputPreprocessor:

738 request_id: str,

739 lora_request: Optional[LoRARequest] = None,

740 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

741 + return_mm_hashes: bool = False,

742 ) -> DecoderOnlyInputs:

743 """Async version of :meth:`_process_decoder_only_prompt`."""

744 prompt_comps = await self._prompt_to_llm_inputs_async(

@@ -748,9 +758,13 @@ class InputPreprocessor:

758 request_id: str,

759 lora_request: Optional[LoRARequest] = None,

760 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

761 + return_mm_hashes: bool = False,

762 ) -> ProcessorInputs:

763 """Preprocess the input prompt."""

764 if self.model_config.is_encoder_decoder:

765 + assert not return_mm_hashes, (

766 + "Multimodal hashes for encoder-decoder models should not be ",

767 + "returned until they are supported on vLLM V1.")

768 # Encoder-decoder model requires special mapping of

769 # input prompts to encoder & decoder

770 return self._process_encoder_decoder_prompt(

@@ -768,6 +782,7 @@ class InputPreprocessor:

782 request_id=request_id,

783 lora_request=lora_request,

784 prompt_adapter_request=prompt_adapter_request,

785 + return_mm_hashes=return_mm_hashes,

786 )

787

788 async def preprocess_async(

@@ -776,9 +791,13 @@ class InputPreprocessor:

791 request_id: str,

792 lora_request: Optional[LoRARequest] = None,

793 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

794 + return_mm_hashes: bool = False,

795 ) -> ProcessorInputs:

796 """Async version of :meth:`preprocess`."""

797 if self.model_config.is_encoder_decoder:

798 + assert not return_mm_hashes, (

799 + "Multimodal hashes for encoder-decoder models should not be ",

800 + "returned until they are supported on vLLM V1.")

801 # Encoder-decoder model requires special mapping of

802 # input prompts to encoder & decoder

803 return await self._process_encoder_decoder_prompt_async(

@@ -796,4 +815,5 @@ class InputPreprocessor:

815 request_id=request_id,

816 lora_request=lora_request,

817 prompt_adapter_request=prompt_adapter_request,

818 + return_mm_hashes=return_mm_hashes,

819 )

vllm/model_executor/models/llava.py

+3 -1

@@ -767,6 +767,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

767 prompt: Union[str, list[int]],

768 mm_data: MultiModalDataDict,

769 hf_processor_mm_kwargs: Mapping[str, object],

770 + return_mm_hashes: bool = False,

771 ) -> MultiModalInputs:

772 hf_config = self.info.get_hf_config()

773 image_token_id = hf_config.image_token_index

@@ -777,7 +778,8 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

778 image_height=-1,

779 )

780

780 - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)

781 + result = super().apply(prompt, mm_data, hf_processor_mm_kwargs,

782 + return_mm_hashes)

783

784 mm_items = self._to_mm_items(mm_data)

785 mm_item_counts = mm_items.get_all_counts()

vllm/model_executor/models/minicpmv.py

+3 -1

@@ -780,6 +780,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

780 prompt: Union[str, List[int]],

781 mm_data: MultiModalDataDict,

782 hf_processor_mm_kwargs: Mapping[str, object],

783 + return_mm_hashes: bool = False,

784 ) -> MultiModalInputs:

785 supported_mm_modalities = self.info.get_supported_mm_modalities()

786 if isinstance(prompt, list):

@@ -791,7 +792,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

792 [index for index, m in enumerate(matches) if m == modality])

793 for modality in supported_mm_modalities

794 }

794 - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)

795 + result = super().apply(prompt, mm_data, hf_processor_mm_kwargs,

796 + return_mm_hashes)

797 # Exclude <image_id>x</image_id> from placeholders

798 if "image" in result["mm_placeholders"] and \

799 self.info.get_model_version() == (2, 6):

vllm/model_executor/models/mllama.py

+3 -1

@@ -175,8 +175,10 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]

175 prompt: Union[str, list[int]],

176 mm_data: MultiModalDataDict,

177 hf_processor_mm_kwargs: Mapping[str, object],

178 + return_mm_hashes: bool = False,

179 ) -> MultiModalEncDecInputs:

179 - mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs)

180 + mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,

181 + return_mm_hashes)

182

183 # Check that the number of image tokens in the decoder prompt matches

184 # the number of images provided in mm_data

vllm/model_executor/models/prithvi_geospatial_mae.py

@@ -93,6 +93,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):

93 prompt: Union[str, list[int]],

94 mm_data: MultiModalDataDict,

95 hf_processor_mm_kwargs: Mapping[str, object],

96 + return_mm_hashes: bool = False,

97 ) -> MultiModalInputs:

98 mm_kwargs = {}

vllm/multimodal/processing.py

+5 -3

@@ -14,7 +14,6 @@ from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,

14 from transformers import BatchFeature, PretrainedConfig, ProcessorMixin

15 from typing_extensions import assert_never

17 -import vllm.envs as envs

17 from vllm.inputs import InputProcessingContext

18 from vllm.logger import init_logger

19 from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,

@@ -1435,6 +1434,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1434 prompt: Union[str, list[int]],

1435 mm_data: MultiModalDataDict,

1436 hf_processor_mm_kwargs: Mapping[str, object],

1437 + return_mm_hashes: bool = False,

1438 ) -> MultiModalInputs:

1439 """

1440 Process multi-modal inputs to be used in vLLM.

@@ -1451,11 +1451,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1451 """

1452 mm_items = self._to_mm_items(mm_data)

1453

1454 - # Create MM hashes (only used in V1)

1454 + # Create MM hashes to be returned (only used in V1)

1455 # TODO: Use these hash keys for caching operations in apply_hf_processor

1456 # instead of rehashing.

1457

1458 - if envs.VLLM_USE_V1:

1458 + if return_mm_hashes:

1459 model_id = self.info.model_id

1460 mm_hashes = {

1461 modality: [

@@ -1554,6 +1554,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1554 prompt: Union[str, list[int]],

1555 mm_data: MultiModalDataDict,

1556 hf_processor_mm_kwargs: Mapping[str, object],

1557 + return_mm_hashes: bool = False,

1558 ) -> MultiModalEncDecInputs:

1559 """

1560 Process multi-modal inputs to be used in vLLM.

@@ -1567,6 +1568,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1568 encoder_prompt,

1569 mm_data,

1570 hf_processor_mm_kwargs,

1571 + return_mm_hashes,

1572 )

1573

1574 tokenizer = self.info.get_tokenizer()

vllm/v1/engine/processor.py

@@ -131,6 +131,7 @@ class Processor:

131 request_id=request_id,

132 lora_request=lora_request,

133 prompt_adapter_request=prompt_adapter_request,

134 + return_mm_hashes=self.use_hash,

135 )

136 eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)

137