A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] Replace `is_encoder_decoder_inputs` with `split_enc_dec_inputs` (#15620)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 28, 2025 at 01:36 UTC
247181536fc2cab728077f3e7489622e19671d2d
8 files changed
+47
-52
tests/models/multimodal/processing/test_idefics3.py
+1
-1
@@ -29,7 +29,7 @@ def test_processor_override(
29
num_imgs: int,
30
kwargs_on_init: bool,
31
):
32
- """Ensure input_processor_for_idefics3 handles num_crops properly."""
32
+ """Ensure Idefics3MultiModalProcessor handles num_crops properly."""
33
# Same as the previous test - don't initialize mm_processor_kwargs
34
# in this test and assume that the kwargs will be correctly expanded by
35
# the partial when calling the custom input processor.
tests/models/multimodal/processing/test_phi3v.py
+1
-1
@@ -30,7 +30,7 @@ def test_processor_override(
30
num_imgs: int,
31
kwargs_on_init: bool,
32
):
33
- """Ensure input_processor_for_phi3v handles num_crops properly."""
33
+ """Ensure Phi3VMultiModalProcessor handles num_crops properly."""
34
# Avoid initializing CUDA early
35
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
36
vllm/engine/arg_utils.py
+1
-1
@@ -665,7 +665,7 @@ class EngineArgs:
665
type=nullable_kvs,
666
default=EngineArgs.limit_mm_per_prompt,
667
# The default value is given in
668
- # MultiModalRegistry.init_mm_limits_per_prompt
668
+ # MultiModalConfig.get_limit_per_prompt
669
help=('For each multimodal plugin, limit how many '
670
'input instances to allow for each prompt. '
671
'Expects a comma-separated list of items, '
vllm/engine/llm_engine.py
+11
-15
@@ -30,8 +30,8 @@ from vllm.entrypoints.openai.logits_processors import (
30
get_logits_processors as get_openai_logits_processors)
31
from vllm.executor.executor_base import ExecutorBase
32
from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
33
- PromptType, SingletonInputsAdapter)
34
-from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt
33
+ PromptType)
34
+from vllm.inputs.parse import is_token_prompt, split_enc_dec_inputs
35
from vllm.inputs.preprocess import InputPreprocessor
36
from vllm.logger import init_logger
37
from vllm.logits_process import get_bad_words_logits_processors
@@ -609,12 +609,7 @@ class LLMEngine:
609
seq_id = next(self.seq_counter)
610
eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
611
612
- if is_encoder_decoder_inputs(processed_inputs):
613
- decoder_inputs = processed_inputs["decoder"]
614
- encoder_inputs = processed_inputs["encoder"]
615
- else:
616
- decoder_inputs = processed_inputs
617
- encoder_inputs = None
612
+ encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
613
614
seq = Sequence(seq_id, decoder_inputs, block_size, eos_token_id,
615
lora_request, prompt_adapter_request)
@@ -2031,15 +2026,16 @@ class LLMEngine:
2026
2027
def _validate_model_inputs(self, inputs: ProcessorInputs,
2028
lora_request: Optional[LoRARequest]):
2034
- if is_encoder_decoder_inputs(inputs):
2035
- # For encoder-decoder multimodal models, the max_prompt_len
2036
- # restricts the decoder prompt length
2037
- prompt_inputs = inputs["decoder" if self.model_config.
2038
- is_multimodal_model else "encoder"]
2029
+ encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs)
2030
+
2031
+ # For encoder-decoder multimodal models, the max_prompt_len
2032
+ # restricts the decoder prompt length
2033
+ if self.model_config.is_multimodal_model:
2034
+ prompt_inputs = decoder_inputs
2035
else:
2040
- prompt_inputs = inputs
2036
+ prompt_inputs = encoder_inputs or decoder_inputs
2037
2042
- prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids
2038
+ prompt_ids = prompt_inputs["prompt_token_ids"]
2039
2040
if prompt_ids is None or len(prompt_ids) == 0:
2041
raise ValueError("Prompt cannot be empty")
vllm/inputs/parse.py
+14
-8
@@ -1,15 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-
2
from collections.abc import Sequence
4
-from typing import Literal, TypedDict, Union, cast, overload
3
+from typing import Literal, Optional, TypedDict, Union, cast, overload
4
5
from typing_extensions import TypeIs
6
7
from vllm.utils import is_list_of
8
10
-from .data import (EncoderDecoderInputs, ExplicitEncoderDecoderPrompt,
11
- ProcessorInputs, PromptType, SingletonPrompt, TextPrompt,
12
- TokensPrompt)
9
+from .data import (ExplicitEncoderDecoderPrompt, ProcessorInputs, PromptType,
10
+ SingletonInputs, SingletonPrompt, TextPrompt, TokensPrompt)
11
12
13
class ParsedText(TypedDict):
@@ -110,6 +108,14 @@ def is_explicit_encoder_decoder_prompt(
108
return isinstance(prompt, dict) and "encoder_prompt" in prompt
109
110
113
-def is_encoder_decoder_inputs(
114
- inputs: ProcessorInputs) -> TypeIs[EncoderDecoderInputs]:
115
- return "encoder" in inputs and "decoder" in inputs
111
+def split_enc_dec_inputs(
112
+ inputs: ProcessorInputs,
113
+) -> tuple[Optional[SingletonInputs], SingletonInputs]:
114
+ if "encoder" in inputs and "decoder" in inputs:
115
+ # NOTE: This passes pyright but not mypy
116
+ return (
117
+ inputs["encoder"], # type: ignore[typeddict-item]
118
+ inputs["decoder"], # type: ignore[typeddict-item]
119
+ )
120
+
121
+ return None, inputs
vllm/inputs/registry.py
+6
-8
@@ -19,7 +19,7 @@ from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
19
resolve_mm_processor_kwargs)
20
21
from .data import ProcessorInputs, SingletonInputs
22
-from .parse import is_encoder_decoder_inputs
22
+from .parse import split_enc_dec_inputs
23
24
if TYPE_CHECKING:
25
from vllm.config import ModelConfig
@@ -462,13 +462,11 @@ class InputRegistry:
462
**mm_processor_kwargs,
463
)
464
465
- if is_encoder_decoder_inputs(processed_inputs):
466
- self._ensure_mm_kwargs(processed_inputs["encoder"],
467
- mm_processor_kwargs)
468
- self._ensure_mm_kwargs(processed_inputs["decoder"],
469
- mm_processor_kwargs)
470
- else:
471
- self._ensure_mm_kwargs(processed_inputs, mm_processor_kwargs)
465
+ encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
466
+ if encoder_inputs is not None:
467
+ self._ensure_mm_kwargs(encoder_inputs, mm_processor_kwargs)
468
+ if decoder_inputs is not None:
469
+ self._ensure_mm_kwargs(decoder_inputs, mm_processor_kwargs)
470
471
return processed_inputs
472
vllm/model_executor/models/idefics3.py
+2
-2
@@ -232,7 +232,7 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]
232
)
233
234
235
-class Idefics3MultimodalProcessor(
235
+class Idefics3MultiModalProcessor(
236
BaseMultiModalProcessor[Idefics3ProcessingInfo]):
237
238
def _call_hf_processor(
@@ -575,7 +575,7 @@ class Idefics3Model(nn.Module):
575
576
577
@MULTIMODAL_REGISTRY.register_processor(
578
- Idefics3MultimodalProcessor,
578
+ Idefics3MultiModalProcessor,
579
info=Idefics3ProcessingInfo,
580
dummy_inputs=Idefics3DummyInputsBuilder)
581
class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
vllm/v1/engine/processor.py
+11
-16
@@ -7,7 +7,7 @@ from typing import Optional, Union
7
from vllm.config import VllmConfig
8
from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
9
PromptType, SingletonInputsAdapter)
10
-from vllm.inputs.parse import is_encoder_decoder_inputs
10
+from vllm.inputs.parse import split_enc_dec_inputs
11
from vllm.inputs.preprocess import InputPreprocessor
12
from vllm.lora.request import LoRARequest
13
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
@@ -209,14 +209,8 @@ class Processor:
209
210
self._validate_model_inputs(processed_inputs, lora_request)
211
212
- if is_encoder_decoder_inputs(processed_inputs):
213
- decoder_inputs = SingletonInputsAdapter(
214
- processed_inputs["decoder"])
215
- encoder_inputs = SingletonInputsAdapter(
216
- processed_inputs["encoder"])
217
- else:
218
- decoder_inputs = SingletonInputsAdapter(processed_inputs)
219
- encoder_inputs = None
212
+ encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
213
+ decoder_inputs = SingletonInputsAdapter(decoder_inputs)
214
215
# TODO: Impl encoder-decoder
216
if encoder_inputs is not None:
@@ -301,15 +295,16 @@ class Processor:
295
def _validate_model_inputs(self,
296
inputs: ProcessorInputs,
297
lora_request: Optional[LoRARequest] = None):
304
- if is_encoder_decoder_inputs(inputs):
305
- # For encoder-decoder multimodal models, the max_prompt_len
306
- # restricts the decoder prompt length
307
- prompt_inputs = inputs["decoder" if self.model_config.
308
- is_multimodal_model else "encoder"]
298
+ encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs)
299
+
300
+ # For encoder-decoder multimodal models, the max_prompt_len
301
+ # restricts the decoder prompt length
302
+ if self.model_config.is_multimodal_model:
303
+ prompt_inputs = decoder_inputs
304
else:
310
- prompt_inputs = inputs
305
+ prompt_inputs = encoder_inputs or decoder_inputs
306
312
- prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids
307
+ prompt_ids = prompt_inputs["prompt_token_ids"]
308
309
if prompt_ids is None or len(prompt_ids) == 0:
310
raise ValueError("Prompt cannot be empty")