ba2f0acc2d · tduncan/Nvidia-Optimization-Stack-VLLM

docs/api/README.md

+3 -12

@@ -27,11 +27,9 @@ LLM Class.

27

28 - [vllm.LLM][]

29

30 -LLM Inputs.

30 +Prompt schema for LLM APIs.

31

32 -- [vllm.inputs.PromptType][]

33 -- [vllm.inputs.TextPrompt][]

34 -- [vllm.inputs.TokensPrompt][]

32 +- [vllm.inputs.llm][]

33

34 ## vLLM Engines

35

@@ -58,13 +56,7 @@ Looking to add your own multi-modal model? Please follow the instructions listed

56

57 - [vllm.multimodal.MULTIMODAL_REGISTRY][]

58

61 -### Inputs

62 -

63 -User-facing inputs.

64 -

65 -- [vllm.multimodal.inputs.MultiModalDataDict][]

66 -

67 -Internal data structures.

59 +### Internal data structures

60

61 - [vllm.multimodal.inputs.PlaceholderRange][]

62 - [vllm.multimodal.inputs.NestedTensors][]

@@ -72,7 +64,6 @@ Internal data structures.

64 - [vllm.multimodal.inputs.MultiModalFieldConfig][]

65 - [vllm.multimodal.inputs.MultiModalKwargsItem][]

66 - [vllm.multimodal.inputs.MultiModalKwargsItems][]

75 -- [vllm.multimodal.inputs.MultiModalInputs][]

67

68 ### Data Parsing

69

docs/contributing/model/transcription.md

+2 -2

@@ -23,7 +23,7 @@ Declare supported languages and capabilities:

23 from torch import nn

24

25 from vllm.config import ModelConfig, SpeechToTextConfig

26 - from vllm.inputs.data import PromptType

26 + from vllm.inputs import PromptType

27 from vllm.model_executor.models.interfaces import SupportsTranscription

28

29 class YourASRModel(nn.Module, SupportsTranscription):

@@ -66,7 +66,7 @@ This is for controlling general behavior of the API when serving your model:

66

67 See [Audio preprocessing and chunking](#audio-preprocessing-and-chunking) for what each field controls.

68

69 -Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server passes you the resampled waveform and task parameters; you return a valid [PromptType][vllm.inputs.data.PromptType]. There are two common patterns:

69 +Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server passes you the resampled waveform and task parameters; you return a valid [PromptType][vllm.inputs.llm.PromptType]. There are two common patterns:

70

71 #### Multimodal LLM with audio embeddings (e.g., Voxtral, Gemma3n)

72

docs/features/multimodal_inputs.md

+1 -1

@@ -18,7 +18,7 @@ This page teaches you how to pass multi-modal inputs to [multi-modal models](../

18 To input multi-modal data, follow this schema in [vllm.inputs.PromptType][]:

19

20 - `prompt`: The prompt should follow the format that is documented on HuggingFace.

21 -- `multi_modal_data`: This is a dictionary that follows the schema defined in [vllm.multimodal.inputs.MultiModalDataDict][].

21 +- `multi_modal_data`: This is a dictionary that follows the schema defined in [vllm.inputs.MultiModalDataDict][].

22

23 ### Image Inputs

24

examples/pooling/token_embed/jina_embeddings_v4_offline.py

+1 -1

@@ -4,7 +4,7 @@

4 import torch

5

6 from vllm import LLM

7 -from vllm.inputs.data import TextPrompt

7 +from vllm.inputs import TextPrompt

8 from vllm.multimodal.utils import fetch_image

9

10 # Initialize model

tests/entrypoints/openai/chat_completion/test_chat_error.py

+1 -1

@@ -105,7 +105,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:

105 )

106

107 async def _fake_preprocess_chat(*args, **kwargs):

108 - # return conversation, engine_prompts

108 + # return conversation, engine_inputs

109 return (

110 [{"role": "user", "content": "Test"}],

111 [{"prompt_token_ids": [1, 2, 3]}],

tests/entrypoints/openai/chat_completion/test_serving_chat.py

+8 -8

@@ -958,14 +958,14 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

958 serving_chat = _build_serving_chat(mock_engine)

959

960 orig_render_chat_request = serving_chat.render_chat_request

961 - captured_prompts = []

961 + captured_inputs = []

962

963 async def render_chat_request(request):

964 result = await orig_render_chat_request(request)

965

966 assert isinstance(result, tuple)

967 - conversation, engine_prompts = result

968 - captured_prompts.extend(engine_prompts)

967 + conversation, engine_inputs = result

968 + captured_inputs.extend(engine_inputs)

969

970 return result

971

@@ -981,18 +981,18 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

981 with suppress(Exception):

982 await serving_chat.create_chat_completion(req)

983

984 - assert len(captured_prompts) == 1

985 - assert "cache_salt" not in captured_prompts[0]

984 + assert len(captured_inputs) == 1

985 + assert "cache_salt" not in captured_inputs[0]

986

987 - captured_prompts.clear()

987 + captured_inputs.clear()

988

989 # Test with certain cache_salt

990 req.cache_salt = "test_salt"

991 with suppress(Exception):

992 await serving_chat.create_chat_completion(req)

993

994 - assert len(captured_prompts) == 1

995 - assert captured_prompts[0]["cache_salt"] == "test_salt"

994 + assert len(captured_inputs) == 1

995 + assert captured_inputs[0]["cache_salt"] == "test_salt"

996

997

998 @pytest.mark.asyncio

tests/entrypoints/openai/responses/test_serving_responses.py

+5 -5

@@ -37,7 +37,7 @@ from vllm.entrypoints.openai.responses.serving import (

37 from vllm.entrypoints.openai.responses.streaming_events import (

38 StreamingState,

39 )

40 -from vllm.inputs.data import TokensPrompt

40 +from vllm.inputs import tokens_input

41 from vllm.outputs import CompletionOutput, RequestOutput

42 from vllm.sampling_params import SamplingParams

43

@@ -258,20 +258,20 @@ class TestValidateGeneratorInput:

258 """Test _validate_generator_input with valid prompt length"""

259 # Create an engine prompt with valid length (less than max_model_len)

260 valid_prompt_token_ids = list(range(5)) # 5 tokens < 100 max_model_len

261 - engine_prompt = TokensPrompt(prompt_token_ids=valid_prompt_token_ids)

261 + engine_input = tokens_input(valid_prompt_token_ids)

262

263 # Call the method

264 - result = serving_responses_instance._validate_generator_input(engine_prompt)

264 + result = serving_responses_instance._validate_generator_input(engine_input)

265

266 # Should return None for valid input

267 assert result is None

268

269 # create an invalid engine prompt

270 invalid_prompt_token_ids = list(range(200)) # 100 tokens >= 100 max_model_len

271 - engine_prompt = TokensPrompt(prompt_token_ids=invalid_prompt_token_ids)

271 + engine_input = tokens_input(invalid_prompt_token_ids)

272

273 # Call the method

274 - result = serving_responses_instance._validate_generator_input(engine_prompt)

274 + result = serving_responses_instance._validate_generator_input(engine_input)

275

276 # Should return an ErrorResponse

277 assert result is not None

tests/entrypoints/serve/render/test_launch_render.py

-14

@@ -73,20 +73,6 @@ async def test_chat_render_multi_turn(client):

73 assert len(data["token_ids"]) > 0

74

75

76 -@pytest.mark.asyncio

77 -async def test_chat_render_invalid_model(client):

78 - response = await client.post(

79 - "/v1/chat/completions/render",

80 - json={

81 - "model": "nonexistent-model",

82 - "messages": [{"role": "user", "content": "Hello"}],

83 - },

84 - )

85 -

86 - assert response.status_code == 404

87 - assert "error" in response.json()

88 -

89 -

76 # -- Completion Render --

77

78

tests/entrypoints/test_chat_utils.py

+1 -1

@@ -16,7 +16,7 @@ from vllm.entrypoints.chat_utils import (

16 parse_chat_messages,

17 parse_chat_messages_async,

18 )

19 -from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict

19 +from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict

20 from vllm.multimodal.utils import (

21 encode_audio_url,

22 encode_image_url,

tests/models/multimodal/generation/test_pixtral.py

+1 -1

@@ -13,8 +13,8 @@ from mistral_common.tokens.tokenizers.multimodal import image_from_chunk

13 from transformers import AutoProcessor

14

15 from vllm import SamplingParams, TextPrompt, TokensPrompt

16 +from vllm.inputs import MultiModalDataBuiltins

17 from vllm.logprobs import Logprob, SampleLogprobs

17 -from vllm.multimodal import MultiModalDataBuiltins

18 from vllm.platforms import current_platform

19

20 from ....utils import VLLM_PATH, large_gpu_test

tests/models/multimodal/processing/test_common.py

+6 -8

@@ -15,13 +15,11 @@ from vllm.config.multimodal import (

15 ImageDummyOptions,

16 VideoDummyOptions,

17 )

18 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict

18 +from vllm.inputs import MultiModalDataDict, MultiModalInput

19 +from vllm.multimodal import MULTIMODAL_REGISTRY

20 from vllm.multimodal.cache import MultiModalProcessorOnlyCache

20 -from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal

21 -from vllm.multimodal.processing import (

22 - BaseMultiModalProcessor,

23 - InputProcessingContext,

24 -)

21 +from vllm.multimodal.inputs import batched_tensors_equal

22 +from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext

23 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

24 from vllm.utils.mistral import is_mistral_tokenizer

25

@@ -420,8 +418,8 @@ def test_processing_correctness(

418

419

420 def _assert_inputs_equal(

423 - a: MultiModalInputs,

424 - b: MultiModalInputs,

421 + a: MultiModalInput,

422 + b: MultiModalInput,

423 *,

424 ignore_mm_keys: set[str] | None = None,

425 msg: str = "",

tests/plugins/bge_m3_sparse_plugin/bge_m3_sparse_processor/sparse_embeddings_processor.py

+2 -4

@@ -6,11 +6,9 @@ from collections.abc import Sequence

6 from vllm.config import ModelConfig, PoolerConfig, VllmConfig

7 from vllm.entrypoints.openai.engine.protocol import UsageInfo

8 from vllm.entrypoints.pooling.base.protocol import EmbedRequestMixin

9 -from vllm.inputs.data import PromptType

9 +from vllm.inputs import PromptType

10 from vllm.outputs import PoolingRequestOutput

11 -from vllm.plugins.io_processors.interface import (

12 - IOProcessor,

13 -)

11 +from vllm.plugins.io_processors.interface import IOProcessor

12 from vllm.pooling_params import PoolingParams

13 from vllm.renderers import BaseRenderer

14 from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens

tests/plugins/prithvi_io_processor_plugin/prithvi_io_processor/prithvi_processor.py

+1 -1

@@ -18,7 +18,7 @@ from einops import rearrange

18 from terratorch.datamodules import Sen1Floods11NonGeoDataModule

19

20 from vllm.config import VllmConfig

21 -from vllm.inputs.data import PromptType

21 +from vllm.inputs import PromptType

22 from vllm.logger import init_logger

23 from vllm.outputs import PoolingRequestOutput

24 from vllm.plugins.io_processors.interface import IOProcessor

tests/plugins_tests/test_io_processor_plugins.py

+1 -1

@@ -6,7 +6,7 @@ from unittest.mock import MagicMock, patch

6 import pytest

7

8 from vllm.config import VllmConfig

9 -from vllm.inputs.data import PromptType

9 +from vllm.inputs import PromptType

10 from vllm.outputs import PoolingRequestOutput

11 from vllm.plugins.io_processors import get_io_processor

12 from vllm.plugins.io_processors.interface import IOProcessor

tests/renderers/inputs/test_preprocess.py

+2 -2

@@ -15,13 +15,13 @@ def test_text_input():

15 assert prompt_to_seq(["foo", "bar"]) == ["foo", "bar"]

16

17

18 -def test_token_input():

18 +def test_tokens_input():

19 assert prompt_to_seq([1, 2]) == [[1, 2]]

20 assert prompt_to_seq([[1, 2]]) == [[1, 2]]

21 assert prompt_to_seq([[1, 2], [3, 4]]) == [[1, 2], [3, 4]]

22

23

24 -def test_text_token_input():

24 +def test_text_tokens_input():

25 assert prompt_to_seq([[1, 2], "foo"]) == [[1, 2], "foo"]

26 assert prompt_to_seq(["foo", [1, 2]]) == ["foo", [1, 2]]

27

tests/renderers/test_completions.py

+2 -2

@@ -129,7 +129,7 @@ class TestValidatePrompt:

129

130

131 class TestRenderPrompt:

132 - def test_token_input(self):

132 + def test_tokens_input(self):

133 renderer = _build_renderer(MockModelConfig())

134

135 tokens = [101, 7592, 2088]

@@ -339,7 +339,7 @@ class TestRenderPrompt:

339 TokenizeParams(max_total_tokens=100),

340 )

341

342 - def test_token_input_with_needs_detokenization(self):

342 + def test_tokens_input_with_needs_detokenization(self):

343 renderer = _build_renderer(MockModelConfig())

344

345 tokens = [1, 2, 3, 4]

tests/v1/shutdown/test_processor_error.py

+1 -1

@@ -9,7 +9,7 @@ import pytest

9 from tests.v1.shutdown.utils import SHUTDOWN_TEST_TIMEOUT_SEC

10 from vllm import SamplingParams

11 from vllm.engine.arg_utils import AsyncEngineArgs

12 -from vllm.inputs.data import TokensPrompt

12 +from vllm.inputs import TokensPrompt

13 from vllm.sampling_params import RequestOutputKind

14 from vllm.v1.engine.async_llm import AsyncLLM

15 from vllm.v1.engine.exceptions import EngineGenerateError

vllm/beam_search.py

+18 -13

@@ -3,11 +3,16 @@

3

4 from dataclasses import dataclass

5

6 -from vllm.inputs import EncoderDecoderInputs, TokenInputs, token_inputs

7 -from vllm.inputs.data import DecoderInputs

6 +from vllm.inputs import (

7 + DecoderOnlyEngineInput,

8 + EncoderDecoderInput,

9 + MultiModalInput,

10 + TokensInput,

11 + mm_input,

12 + tokens_input,

13 +)

14 from vllm.logprobs import Logprob

15 from vllm.lora.request import LoRARequest

10 -from vllm.multimodal.inputs import MultiModalInputs, mm_inputs

16

17

18 @dataclass

@@ -18,7 +23,7 @@ class BeamSearchSequence:

23 about to be returned to the user.

24 """

25

21 - orig_prompt: TokenInputs | MultiModalInputs | EncoderDecoderInputs

26 + orig_prompt: TokensInput | MultiModalInput | EncoderDecoderInput

27

28 # NOTE: Tokens represents decoder tokens in the encoder / decoder case

29 tokens: list[int]

@@ -40,13 +45,13 @@ class BeamSearchSequence:

45 cache_salt = prompt.get("cache_salt")

46

47 if prompt["type"] == "token":

43 - return token_inputs(

48 + return tokens_input(

49 self.tokens,

50 prompt=prompt_text,

51 cache_salt=cache_salt,

52 )

53

49 - return mm_inputs(

54 + return mm_input(

55 prompt_token_ids=self.tokens,

56 mm_kwargs=prompt["mm_kwargs"],

57 mm_hashes=prompt["mm_hashes"],

@@ -56,8 +61,8 @@ class BeamSearchSequence:

61 )

62

63 def _build_encoder_decoder_inputs(

59 - self, prompt: EncoderDecoderInputs

60 - ) -> EncoderDecoderInputs:

64 + self, prompt: EncoderDecoderInput

65 + ) -> EncoderDecoderInput:

66 """Rebuild the encoder-decoder inputs with the current beam search

67 sequence's tokens.

68

@@ -70,9 +75,9 @@ class BeamSearchSequence:

75

76 # Rebuild decoder prompt with updated tokens,

77 # but keep everything else the same.

73 - new_dec_prompt: DecoderInputs

78 + new_dec_prompt: DecoderOnlyEngineInput

79 if dec_prompt["type"] == "multimodal":

75 - new_dec_prompt = mm_inputs(

80 + new_dec_prompt = mm_input(

81 self.tokens,

82 mm_kwargs=dec_prompt["mm_kwargs"],

83 mm_hashes=dec_prompt["mm_hashes"],

@@ -81,13 +86,13 @@ class BeamSearchSequence:

86 cache_salt=dec_prompt.get("cache_salt"),

87 )

88 else:

84 - new_dec_prompt = token_inputs(

89 + new_dec_prompt = tokens_input(

90 self.tokens,

91 prompt=dec_prompt.get("prompt"),

92 cache_salt=dec_prompt.get("cache_salt"),

93 )

94

90 - return EncoderDecoderInputs(

95 + return EncoderDecoderInput(

96 type="enc_dec",

97 encoder_prompt=prompt["encoder_prompt"],

98 decoder_prompt=new_dec_prompt,

@@ -107,7 +112,7 @@ class BeamSearchOutput:

112 class BeamSearchInstance:

113 def __init__(

114 self,

110 - prompt: TokenInputs | MultiModalInputs | EncoderDecoderInputs,

115 + prompt: TokensInput | MultiModalInput | EncoderDecoderInput,

116 lora_request: LoRARequest | None = None,

117 logprobs: list[dict[int, Logprob]] | None = None,

118 **kwargs,

vllm/benchmarks/datasets.py

+1 -1

@@ -35,9 +35,9 @@ from huggingface_hub import snapshot_download

35 from PIL import Image

36 from typing_extensions import deprecated

37

38 +from vllm.inputs import MultiModalDataDict

39 from vllm.lora.request import LoRARequest

40 from vllm.lora.utils import get_adapter_absolute_path

40 -from vllm.multimodal import MultiModalDataDict

41 from vllm.multimodal.audio import get_audio_duration

42 from vllm.multimodal.image import convert_image_mode

43 from vllm.tokenizers import TokenizerLike

vllm/engine/protocol.py

+4 -4

@@ -11,7 +11,7 @@ from vllm.distributed.weight_transfer.base import (

11 WeightTransferInitRequest,

12 WeightTransferUpdateRequest,

13 )

14 -from vllm.inputs.data import ProcessorInputs, PromptType

14 +from vllm.inputs import EngineInput, PromptType

15 from vllm.lora.request import LoRARequest

16 from vllm.outputs import PoolingRequestOutput, RequestOutput

17 from vllm.plugins.io_processors import IOProcessor

@@ -34,7 +34,7 @@ class StreamingInput:

34 where inputs are provided via an async generator.

35 """

36

37 - prompt: ProcessorInputs

37 + prompt: EngineInput

38 sampling_params: SamplingParams | None = None

39

40

@@ -68,7 +68,7 @@ class EngineClient(ABC):

68 self,

69 prompt: EngineCoreRequest

70 | PromptType

71 - | ProcessorInputs

71 + | EngineInput

72 | AsyncGenerator[StreamingInput, None],

73 sampling_params: SamplingParams,

74 request_id: str,

@@ -87,7 +87,7 @@ class EngineClient(ABC):

87 @abstractmethod

88 def encode(

89 self,

90 - prompt: PromptType | ProcessorInputs,

90 + prompt: PromptType | EngineInput,

91 pooling_params: PoolingParams,

92 request_id: str,

93 lora_request: LoRARequest | None = None,

vllm/entrypoints/anthropic/serving.py

+4 -4

@@ -797,12 +797,12 @@ class AnthropicServingMessages(OpenAIServingChat):

797 if isinstance(result, ErrorResponse):

798 return result

799

800 - _, engine_prompts = result

800 + _, engine_inputs = result

801

802 input_tokens = sum( # type: ignore

803 - len(prompt["prompt_token_ids"]) # type: ignore[typeddict-item, misc]

804 - for prompt in engine_prompts

805 - if "prompt_token_ids" in prompt

803 + len(engine_input["prompt_token_ids"]) # type: ignore[typeddict-item, misc]

804 + for engine_input in engine_inputs

805 + if "prompt_token_ids" in engine_input

806 )

807

808 response = AnthropicCountTokensResponse(

vllm/entrypoints/chat_utils.py

+2 -1

@@ -40,9 +40,10 @@ from typing_extensions import Required, TypedDict

40

41 from vllm import envs

42 from vllm.config import ModelConfig

43 +from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict

44 from vllm.logger import init_logger

45 from vllm.model_executor.models import SupportsMultiModal

45 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict

46 +from vllm.multimodal import MULTIMODAL_REGISTRY

47 from vllm.multimodal.inputs import (

48 MultiModalBatchedField,

49 MultiModalFlatField,

vllm/entrypoints/llm.py

+22 -22

@@ -57,9 +57,9 @@ from vllm.entrypoints.pooling.score.utils import (

57 validate_score_input,

58 )

59 from vllm.entrypoints.utils import log_non_default_args

60 -from vllm.inputs.data import (

60 +from vllm.inputs import (

61 DataPrompt,

62 - ProcessorInputs,

62 + EngineInput,

63 PromptType,

64 SingletonPrompt,

65 TextPrompt,

@@ -589,7 +589,7 @@ class LLM:

589

590 def _resolve_mm_lora(

591 self,

592 - prompt: ProcessorInputs,

592 + prompt: EngineInput,

593 lora_request: LoRARequest | None,

594 ) -> LoRARequest | None:

595 if prompt["type"] != "multimodal":

@@ -716,8 +716,8 @@ class LLM:

716 eos_token_id = tokenizer.eos_token_id

717 sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)

718

719 - engine_prompts = self._preprocess_cmpl(prompts)

720 - lora_requests = self._lora_request_to_seq(lora_request, len(engine_prompts))

719 + engine_inputs = self._preprocess_cmpl(prompts)

720 + lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))

721

722 if use_tqdm and concurrency_limit is not None:

723 logger.warning(

@@ -727,7 +727,7 @@ class LLM:

727 use_tqdm = False

728

729 if concurrency_limit is None:

730 - concurrency_limit = len(engine_prompts)

730 + concurrency_limit = len(engine_inputs)

731

732 # generate 2 * beam_width candidates at each step

733 # following the huggingface transformers implementation

@@ -740,7 +740,7 @@ class LLM:

740 )

741 instances: list[BeamSearchInstance] = []

742

743 - for lora_req, prompt in zip(lora_requests, engine_prompts):

743 + for lora_req, prompt in zip(lora_requests, engine_inputs):

744 if prompt["type"] == "embeds":

745 raise NotImplementedError(

746 "Embedding prompt not supported for beam search"

@@ -845,7 +845,7 @@ class LLM:

845 self,

846 prompts: Sequence[PromptType],

847 tokenization_kwargs: dict[str, Any] | None = None,

848 - ) -> Sequence[ProcessorInputs]:

848 + ) -> Sequence[EngineInput]:

849 """

850 Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into

851 a format that can be passed to `_add_request`.

@@ -853,7 +853,7 @@ class LLM:

853 Refer to [LLM.generate][] for a complete description of the arguments.

854

855 Returns:

856 - A list of `ProcessorInputs` objects ready to be passed into LLMEngine.

856 + A list of `EngineInput` objects ready to be passed into LLMEngine.

857 """

858 renderer = self.renderer

859 model_config = self.model_config

@@ -871,9 +871,9 @@ class LLM:

871 self,

872 prompt: PromptType,

873 tokenization_kwargs: dict[str, Any] | None = None,

874 - ) -> ProcessorInputs:

875 - (engine_prompt,) = self._preprocess_cmpl([prompt], tokenization_kwargs)

876 - return engine_prompt

874 + ) -> EngineInput:

875 + (engine_input,) = self._preprocess_cmpl([prompt], tokenization_kwargs)

876 + return engine_input

877

878 def _preprocess_chat(

879 self,

@@ -886,7 +886,7 @@ class LLM:

886 tools: list[dict[str, Any]] | None = None,

887 tokenization_kwargs: dict[str, Any] | None = None,

888 mm_processor_kwargs: dict[str, Any] | None = None,

889 - ) -> Sequence[ProcessorInputs]:

889 + ) -> Sequence[EngineInput]:

890 """

891 Convert a list of conversations into prompts so that they can then

892 be used as input for other LLM APIs.

@@ -894,7 +894,7 @@ class LLM:

894 Refer to [LLM.chat][] for a complete description of the arguments.

895

896 Returns:

897 - A list of `ProcessorInputs` objects ready to be passed into LLMEngine.

897 + A list of `EngineInput` objects ready to be passed into LLMEngine.

898 """

899 renderer = self.renderer

900

@@ -915,14 +915,14 @@ class LLM:

915 **(tokenization_kwargs or {})

916 )

917

918 - _, engine_prompts = renderer.render_chat(

918 + _, engine_inputs = renderer.render_chat(

919 conversations,

920 chat_params,

921 tok_params,

922 prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},

923 )

924

925 - return engine_prompts

925 + return engine_inputs

926

927 def _preprocess_chat_one(

928 self,

@@ -935,8 +935,8 @@ class LLM:

935 tools: list[dict[str, Any]] | None = None,

936 tokenization_kwargs: dict[str, Any] | None = None,

937 mm_processor_kwargs: dict[str, Any] | None = None,

938 - ) -> ProcessorInputs:

939 - (engine_prompt,) = self._preprocess_chat(

938 + ) -> EngineInput:

939 + (engine_input,) = self._preprocess_chat(

940 [conversation],

941 chat_template=chat_template,

942 chat_template_content_format=chat_template_content_format,

@@ -948,7 +948,7 @@ class LLM:

948 mm_processor_kwargs=mm_processor_kwargs,

949 )

950

951 - return engine_prompt

951 + return engine_input

952

953 def chat(

954 self,

@@ -1909,7 +1909,7 @@ class LLM:

1909

1910 def _render_and_run_requests(

1911 self,

1912 - prompts: Iterable[ProcessorInputs],

1912 + prompts: Iterable[EngineInput],

1913 params: Sequence[SamplingParams | PoolingParams],

1914 output_type: type[_O],

1915 *,

@@ -1938,7 +1938,7 @@ class LLM:

1938

1939 def _render_and_add_requests(

1940 self,

1941 - prompts: Iterable[ProcessorInputs],

1941 + prompts: Iterable[EngineInput],

1942 params: Sequence[SamplingParams | PoolingParams],

1943 *,

1944 lora_requests: Sequence[LoRARequest | None] | None = None,

@@ -1967,7 +1967,7 @@ class LLM:

1967

1968 def _add_request(

1969 self,

1970 - prompt: ProcessorInputs,

1970 + prompt: EngineInput,

1971 params: SamplingParams | PoolingParams,

1972 lora_request: LoRARequest | None = None,

1973 priority: int = 0,

vllm/entrypoints/openai/chat_completion/serving.py

+11 -11

@@ -63,7 +63,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (

63 )

64 from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls

65 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

66 -from vllm.inputs.data import ProcessorInputs

66 +from vllm.inputs import EngineInput

67 from vllm.logger import init_logger

68 from vllm.logprobs import Logprob

69 from vllm.outputs import CompletionOutput, RequestOutput

@@ -177,7 +177,7 @@ class OpenAIServingChat(OpenAIServing):

177 async def render_chat_request(

178 self,

179 request: ChatCompletionRequest,

180 - ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:

180 + ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:

181 """

182 Validate the model and preprocess a chat completion request.

183

@@ -185,7 +185,7 @@ class OpenAIServingChat(OpenAIServing):

185 engine-aware checks (LoRA model validation, engine health).

186

187 Returns:

188 - A tuple of (conversation, engine_prompts) on success,

188 + A tuple of (conversation, engine_inputs) on success,

189 or an ErrorResponse on failure.

190 """

191 error_check_ret = await self._check_model(request)

@@ -231,7 +231,7 @@ class OpenAIServingChat(OpenAIServing):

231 if isinstance(result, ErrorResponse):

232 return result

233

234 - conversation, engine_prompts = result

234 + conversation, engine_inputs = result

235

236 request_id = (

237 f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"

@@ -251,13 +251,13 @@ class OpenAIServingChat(OpenAIServing):

251 # Schedule the request and get the result generator.

252 max_model_len = self.model_config.max_model_len

253 generators: list[AsyncGenerator[RequestOutput, None]] = []

254 - for i, engine_prompt in enumerate(engine_prompts):

255 - prompt_token_ids = self._extract_prompt_components(engine_prompt).token_ids

254 + for i, engine_input in enumerate(engine_inputs):

255 + prompt_token_ids = self._extract_prompt_components(engine_input).token_ids

256

257 # If we are creating sub requests for multiple prompts, ensure that they

258 # have unique request ids.

259 sub_request_id = (

260 - request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"

260 + request_id if len(engine_inputs) == 1 else f"{request_id}_{i}"

261 )

262

263 max_tokens = get_max_tokens(

@@ -265,7 +265,7 @@ class OpenAIServingChat(OpenAIServing):

265 request.max_completion_tokens

266 if request.max_completion_tokens is not None

267 else request.max_tokens,

268 - self._extract_prompt_len(engine_prompt),

268 + self._extract_prompt_len(engine_input),

269 self.default_sampling_params,

270 self.override_max_tokens,

271 )

@@ -283,7 +283,7 @@ class OpenAIServingChat(OpenAIServing):

283

284 self._log_inputs(

285 sub_request_id,

286 - engine_prompt,

286 + engine_input,

287 params=sampling_params,

288 lora_request=lora_request,

289 )

@@ -296,7 +296,7 @@ class OpenAIServingChat(OpenAIServing):

296

297 if isinstance(sampling_params, BeamSearchParams):

298 generator = self.beam_search(

299 - prompt=engine_prompt,

299 + prompt=engine_input,

300 request_id=sub_request_id,

301 params=sampling_params,

302 lora_request=lora_request,

@@ -313,7 +313,7 @@ class OpenAIServingChat(OpenAIServing):

313 reasoning_ended = None

314

315 generator = self.engine_client.generate(

316 - engine_prompt,

316 + engine_input,

317 sampling_params,

318 sub_request_id,

319 lora_request=lora_request,

vllm/entrypoints/openai/completion/serving.py

+15 -17

@@ -33,7 +33,7 @@ from vllm.entrypoints.openai.engine.serving import (

33 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

34 from vllm.entrypoints.utils import get_max_tokens, should_include_usage

35 from vllm.exceptions import VLLMValidationError

36 -from vllm.inputs.data import ProcessorInputs

36 +from vllm.inputs import EngineInput

37 from vllm.logger import init_logger

38 from vllm.logprobs import Logprob

39 from vllm.outputs import RequestOutput

@@ -82,7 +82,7 @@ class OpenAIServingCompletion(OpenAIServing):

82 async def render_completion_request(

83 self,

84 request: CompletionRequest,

85 - ) -> list[ProcessorInputs] | ErrorResponse:

85 + ) -> list[EngineInput] | ErrorResponse:

86 """

87 Validate the model and preprocess a completion request.

88

@@ -90,8 +90,7 @@ class OpenAIServingCompletion(OpenAIServing):

90 engine-aware checks (LoRA model validation, engine health).

91

92 Returns:

93 - A list of engine_prompts on success,

94 - or an ErrorResponse on failure.

93 + A list of engine_inputs on success, or an ErrorResponse on failure.

94 """

95 error_check_ret = await self._check_model(request)

96 if error_check_ret is not None:

@@ -128,7 +127,7 @@ class OpenAIServingCompletion(OpenAIServing):

127 if isinstance(result, ErrorResponse):

128 return result

129

131 - engine_prompts = result

130 + engine_inputs = result

131

132 request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"

133 created_time = int(time.time())

@@ -145,11 +144,11 @@ class OpenAIServingCompletion(OpenAIServing):

144 # Schedule the request and get the result generator.

145 max_model_len = self.model_config.max_model_len

146 generators: list[AsyncGenerator[RequestOutput, None]] = []

148 - for i, engine_prompt in enumerate(engine_prompts):

147 + for i, engine_input in enumerate(engine_inputs):

148 max_tokens = get_max_tokens(

149 max_model_len,

150 request.max_tokens,

152 - self._extract_prompt_len(engine_prompt),

151 + self._extract_prompt_len(engine_input),

152 self.default_sampling_params,

153 self.override_max_tokens,

154 )

@@ -169,7 +168,7 @@ class OpenAIServingCompletion(OpenAIServing):

168

169 self._log_inputs(

170 request_id_item,

172 - engine_prompt,

171 + engine_input,

172 params=sampling_params,

173 lora_request=lora_request,

174 )

@@ -182,7 +181,7 @@ class OpenAIServingCompletion(OpenAIServing):

181

182 if isinstance(sampling_params, BeamSearchParams):

183 generator = self.beam_search(

185 - prompt=engine_prompt,

184 + prompt=engine_input,

185 request_id=request_id,

186 params=sampling_params,

187 lora_request=lora_request,

@@ -190,7 +189,7 @@ class OpenAIServingCompletion(OpenAIServing):

189 )

190 else:

191 generator = self.engine_client.generate(

193 - engine_prompt,

192 + engine_input,

193 sampling_params,

194 request_id_item,

195 lora_request=lora_request,

@@ -204,7 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):

203 result_generator = merge_async_iterators(*generators)

204

205 model_name = self.models.model_name(lora_request)

207 - num_prompts = len(engine_prompts)

206 + num_prompts = len(engine_inputs)

207

208 # Streaming response

209 tokenizer = self.renderer.tokenizer

@@ -212,7 +211,7 @@ class OpenAIServingCompletion(OpenAIServing):

211 if request.stream:

212 return self.completion_stream_generator(

213 request,

215 - engine_prompts,

214 + engine_inputs,

215 result_generator,

216 request_id,

217 created_time,

@@ -235,8 +234,7 @@ class OpenAIServingCompletion(OpenAIServing):

234 # We did not pass it into vLLM engine to avoid being redundant

235 # with the inputs token IDs

236 if final_res.prompt is None:

238 - engine_prompt = engine_prompts[i]

239 - final_res.prompt = self._extract_prompt_text(engine_prompt)

237 + final_res.prompt = self._extract_prompt_text(engine_inputs[i])

238

239 final_res_batch_checked = cast(list[RequestOutput], final_res_batch)

240

@@ -268,7 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):

266 async def completion_stream_generator(

267 self,

268 request: CompletionRequest,

271 - engine_prompts: list[ProcessorInputs],

269 + engine_inputs: list[EngineInput],

270 result_generator: AsyncIterator[tuple[int, RequestOutput]],

271 request_id: str,

272 created_time: int,

@@ -301,8 +299,8 @@ class OpenAIServingCompletion(OpenAIServing):

299

300 prompt_text = res.prompt

301 if prompt_text is None:

304 - engine_prompt = engine_prompts[prompt_idx]

305 - prompt_text = self._extract_prompt_text(engine_prompt)

302 + engine_input = engine_inputs[prompt_idx]

303 + prompt_text = self._extract_prompt_text(engine_input)

304

305 # Prompt details are excluded from later streamed outputs

306 if prompt_token_ids is not None:

vllm/entrypoints/openai/engine/serving.py

+13 -17

@@ -72,11 +72,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (

72 )

73 from vllm.entrypoints.utils import create_error_response

74 from vllm.exceptions import VLLMValidationError

75 -from vllm.inputs.data import (

76 - ProcessorInputs,

77 - PromptType,

78 - TokensPrompt,

79 -)

75 +from vllm.inputs import EngineInput, PromptType, TokensPrompt

76 from vllm.logger import init_logger

77 from vllm.logprobs import Logprob, PromptLogprobs

78 from vllm.lora.request import LoRARequest

@@ -163,7 +159,7 @@ class ServeContext(Generic[RequestT]):

159 request_id: str

160 created_time: int = field(default_factory=lambda: int(time.time()))

161 lora_request: LoRARequest | None = None

166 - engine_prompts: list[ProcessorInputs] | None = None

162 + engine_inputs: list[EngineInput] | None = None

163

164 result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (

165 None

@@ -202,7 +198,7 @@ class OpenAIServing:

198

199 async def beam_search(

200 self,

205 - prompt: ProcessorInputs,

201 + prompt: EngineInput,

202 request_id: str,

203 params: BeamSearchParams,

204 lora_request: LoRARequest | None = None,

@@ -493,21 +489,21 @@ class OpenAIServing:

489 if isinstance(pooling_params, ErrorResponse):

490 return pooling_params

491

496 - if ctx.engine_prompts is None:

492 + if ctx.engine_inputs is None:

493 return self.create_error_response("Engine prompts not available")

494

499 - for i, engine_prompt in enumerate(ctx.engine_prompts):

495 + for i, engine_input in enumerate(ctx.engine_inputs):

496 request_id_item = f"{ctx.request_id}-{i}"

497

498 self._log_inputs(

499 request_id_item,

504 - engine_prompt,

500 + engine_input,

501 params=pooling_params,

502 lora_request=ctx.lora_request,

503 )

504

505 generator = self.engine_client.encode(

510 - engine_prompt,

506 + engine_input,

507 pooling_params,

508 request_id_item,

509 lora_request=ctx.lora_request,

@@ -526,10 +522,10 @@ class OpenAIServing:

522 ctx: ServeContext,

523 ) -> ErrorResponse | None:

524 """Collect batch results from the result generator."""

529 - if ctx.engine_prompts is None:

525 + if ctx.engine_inputs is None:

526 return self.create_error_response("Engine prompts not available")

527

532 - num_prompts = len(ctx.engine_prompts)

528 + num_prompts = len(ctx.engine_inputs)

529 final_res_batch: list[PoolingRequestOutput | None]

530 final_res_batch = [None] * num_prompts

531

@@ -806,19 +802,19 @@ class OpenAIServing:

802 # Apply server defaults first, then request kwargs override.

803 return default_chat_template_kwargs | request_chat_template_kwargs

804

809 - def _extract_prompt_components(self, prompt: PromptType | ProcessorInputs):

805 + def _extract_prompt_components(self, prompt: PromptType | EngineInput):

806 return extract_prompt_components(self.model_config, prompt)

807

812 - def _extract_prompt_text(self, prompt: ProcessorInputs):

808 + def _extract_prompt_text(self, prompt: PromptType | EngineInput):

809 return self._extract_prompt_components(prompt).text

810

815 - def _extract_prompt_len(self, prompt: ProcessorInputs):

811 + def _extract_prompt_len(self, prompt: EngineInput):

812 return extract_prompt_len(self.model_config, prompt)

813

814 def _log_inputs(

815 self,

816 request_id: str,

821 - inputs: PromptType | ProcessorInputs,

817 + inputs: PromptType | EngineInput,

818 params: SamplingParams | PoolingParams | BeamSearchParams | None,

819 lora_request: LoRARequest | None,

820 ) -> None:

vllm/entrypoints/openai/realtime/serving.py

+3 -3

@@ -12,7 +12,7 @@ from vllm.engine.protocol import EngineClient, StreamingInput

12 from vllm.entrypoints.logger import RequestLogger

13 from vllm.entrypoints.openai.engine.serving import OpenAIServing

14 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

15 -from vllm.inputs.data import PromptType

15 +from vllm.inputs import PromptType

16 from vllm.logger import init_logger

17 from vllm.model_executor.models.interfaces import SupportsRealtime

18 from vllm.renderers.inputs.preprocess import parse_model_prompt

@@ -83,6 +83,6 @@ class OpenAIServingRealtime(OpenAIServing):

83

84 async for prompt in stream_input_iter:

85 parsed_prompt = parse_model_prompt(model_config, prompt)

86 - (engine_prompt,) = await renderer.render_cmpl_async([parsed_prompt])

86 + (engine_input,) = await renderer.render_cmpl_async([parsed_prompt])

87

88 - yield StreamingInput(prompt=engine_prompt)

88 + yield StreamingInput(prompt=engine_input)

vllm/entrypoints/openai/responses/serving.py

+22 -26

@@ -110,7 +110,7 @@ from vllm.entrypoints.openai.responses.utils import (

110 from vllm.entrypoints.serve.render.serving import OpenAIServingRender

111 from vllm.entrypoints.utils import get_max_tokens

112 from vllm.exceptions import VLLMValidationError

113 -from vllm.inputs.data import ProcessorInputs, token_inputs

113 +from vllm.inputs import EngineInput, tokens_input

114 from vllm.logger import init_logger

115 from vllm.logprobs import Logprob as SampleLogprob

116 from vllm.logprobs import SampleLogprobs

@@ -269,10 +269,10 @@ class OpenAIServingResponses(OpenAIServing):

269

270 def _validate_generator_input(

271 self,

272 - engine_prompt: ProcessorInputs,

272 + engine_input: EngineInput,

273 ) -> ErrorResponse | None:

274 """Add validations to the input to the generator here."""

275 - prompt_len = self._extract_prompt_len(engine_prompt)

275 + prompt_len = self._extract_prompt_len(engine_input)

276 max_model_len = self.model_config.max_model_len

277

278 if prompt_len >= max_model_len:

@@ -369,11 +369,11 @@ class OpenAIServingResponses(OpenAIServing):

369 model_name = self.models.model_name(lora_request)

370

371 if self.use_harmony:

372 - messages, engine_prompts = self._make_request_with_harmony(

372 + messages, engine_inputs = self._make_request_with_harmony(

373 request, prev_response

374 )

375 else:

376 - messages, engine_prompts = await self._make_request(request, prev_response)

376 + messages, engine_inputs = await self._make_request(request, prev_response)

377

378 request_metadata = RequestResponseMetadata(request_id=request.request_id)

379 if raw_request:

@@ -413,15 +413,15 @@ class OpenAIServingResponses(OpenAIServing):

413 available_tools = []

414 tokenizer = self.renderer.get_tokenizer()

415

416 - for engine_prompt in engine_prompts:

417 - maybe_error = self._validate_generator_input(engine_prompt)

416 + for engine_input in engine_inputs:

417 + maybe_error = self._validate_generator_input(engine_input)

418 if maybe_error is not None:

419 return maybe_error

420

421 default_max_tokens = get_max_tokens(

422 max_model_len,

423 request.max_output_tokens,

424 - self._extract_prompt_len(engine_prompt),

424 + self._extract_prompt_len(engine_input),

425 self.default_sampling_params,

426 self.override_max_tokens,

427 )

@@ -480,7 +480,7 @@ class OpenAIServingResponses(OpenAIServing):

480 )

481 generator = self._generate_with_builtin_tools(

482 request_id=request.request_id,

483 - engine_prompt=engine_prompt,

483 + engine_input=engine_input,

484 sampling_params=sampling_params,

485 context=context,

486 lora_request=lora_request,

@@ -586,7 +586,7 @@ class OpenAIServingResponses(OpenAIServing):

586 prev_response_output=prev_response.output if prev_response else None,

587 )

588

589 - _, engine_prompts = await self.openai_serving_render.preprocess_chat(

589 + _, engine_inputs = await self.openai_serving_render.preprocess_chat(

590 request,

591 messages,

592 default_template=self.chat_template,

@@ -595,7 +595,7 @@ class OpenAIServingResponses(OpenAIServing):

595 tool_dicts=tool_dicts,

596 tool_parser=self.parser.tool_parser_cls if self.parser else None,

597 )

598 - return messages, engine_prompts

598 + return messages, engine_inputs

599

600 async def _render_next_turn(

601 self,

@@ -610,7 +610,7 @@ class OpenAIServingResponses(OpenAIServing):

610 request_input=messages,

611 )

612

613 - _, engine_prompts = await self.openai_serving_render.preprocess_chat(

613 + _, engine_inputs = await self.openai_serving_render.preprocess_chat(

614 request,

615 new_messages,

616 default_template=chat_template,

@@ -619,12 +619,12 @@ class OpenAIServingResponses(OpenAIServing):

619 tool_dicts=tool_dicts,

620 tool_parser=tool_parser,

621 )

622 - return engine_prompts

622 + return engine_inputs

623

624 async def _generate_with_builtin_tools(

625 self,

626 request_id: str,

627 - engine_prompt: ProcessorInputs,

627 + engine_input: EngineInput,

628 sampling_params: SamplingParams,

629 context: ConversationContext,

630 lora_request: LoRARequest | None = None,

@@ -641,13 +641,13 @@ class OpenAIServingResponses(OpenAIServing):

641

642 self._log_inputs(

643 sub_request_id,

644 - engine_prompt,

644 + engine_input,

645 params=sampling_params,

646 lora_request=lora_request,

647 )

648

649 generator = self.engine_client.generate(

650 - engine_prompt,

650 + engine_input,

651 sampling_params,

652 sub_request_id,

653 lora_request=lora_request,

@@ -675,11 +675,11 @@ class OpenAIServingResponses(OpenAIServing):

675 # Render the next prompt token ids and update sampling_params.

676 if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):

677 token_ids = context.render_for_completion()

678 - engine_prompt = token_inputs(token_ids)

678 + engine_input = tokens_input(token_ids)

679

680 sampling_params.max_tokens = max_model_len - len(token_ids)

681 elif isinstance(context, ParsableContext):

682 - (engine_prompt,) = await self._render_next_turn(

682 + (engine_input,) = await self._render_next_turn(

683 context.request,

684 context.parser.response_messages,

685 context.tool_dicts,

@@ -691,7 +691,7 @@ class OpenAIServingResponses(OpenAIServing):

691 sampling_params.max_tokens = get_max_tokens(

692 max_model_len,

693 context.request.max_output_tokens,

694 - self._extract_prompt_len(engine_prompt),

694 + self._extract_prompt_len(engine_input),

695 self.default_sampling_params, # type: ignore

696 self.override_max_tokens, # type: ignore

697 )

@@ -713,14 +713,10 @@ class OpenAIServingResponses(OpenAIServing):

713 arrival_time = time.time()

714 messages = self._construct_input_messages_with_harmony(request, prev_response)

715 prompt_token_ids = render_for_completion(messages)

716 - engine_prompt = token_inputs(prompt_token_ids)

717 - engine_prompt["arrival_time"] = arrival_time

716 + engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)

717 + engine_input["arrival_time"] = arrival_time

718

719 - # Add cache_salt if provided in the request

720 - if request.cache_salt is not None:

721 - engine_prompt["cache_salt"] = request.cache_salt

722 -

723 - return messages, [engine_prompt]

719 + return messages, [engine_input]

720

721 async def _initialize_tool_sessions(

722 self,

vllm/entrypoints/openai/speech_to_text/speech_to_text.py

+17 -16

@@ -38,7 +38,7 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (

38 )

39 from vllm.entrypoints.utils import get_max_tokens

40 from vllm.exceptions import VLLMValidationError

41 -from vllm.inputs import EncoderDecoderInputs, ProcessorInputs

41 +from vllm.inputs import EncoderDecoderInput, EngineInput

42 from vllm.logger import init_logger

43 from vllm.logprobs import FlatLogprobs, Logprob

44 from vllm.model_executor.models import SupportsTranscription

@@ -171,7 +171,7 @@ class OpenAISpeechToText(OpenAIServing):

171 request: SpeechToTextRequest,

172 audio_data: bytes,

173 request_id: str,

174 - ) -> tuple[list[ProcessorInputs], float]:

174 + ) -> tuple[list[EngineInput], float]:

175 # Validate request

176 language = self.model_cls.validate_language(request.language)

177 # Skip to_language validation to avoid extra logging for Whisper.

@@ -250,9 +250,9 @@ class OpenAISpeechToText(OpenAIServing):

250

251 parsed_prompts.append(parsed_prompt)

252

253 - engine_prompts = await self.renderer.render_cmpl_async(parsed_prompts)

253 + engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)

254

255 - return engine_prompts, duration

255 + return engine_inputs, duration

256

257 def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):

258 dec_prompt = prompt["decoder_prompt"]

@@ -271,7 +271,7 @@ class OpenAISpeechToText(OpenAIServing):

271 return prompt

272

273 @staticmethod

274 - def _get_decoder_prompt_len(engine_prompts: list[ProcessorInputs]) -> int:

274 + def _get_decoder_prompt_len(engine_inputs: list[EngineInput]) -> int:

275 """Get the length of the decoder prompt. Currently we need to offset

276 by the decoder prompt length when running beam search because the mm

277 encoder is not currently cached and runs on decode calls; because of

@@ -282,12 +282,13 @@ class OpenAISpeechToText(OpenAIServing):

282 encoder/decoder caching is implemented.

283 """

284 input_len = 0

285 - assert len(engine_prompts) > 0

286 - first_eng_prompt = engine_prompts[0]

285 + assert len(engine_inputs) > 0

286 + first_input = engine_inputs[0]

287 +

288 + if first_input.get("type") == "enc_dec":

289 + first_input = cast(EncoderDecoderInput, first_input)

290 + input_len = len(first_input["decoder_prompt"]["prompt_token_ids"])

291

288 - if first_eng_prompt.get("type") == "enc_dec":

289 - first_eng_prompt = cast(EncoderDecoderInputs, first_eng_prompt)

290 - input_len = len(first_eng_prompt["decoder_prompt"]["prompt_token_ids"])

292 return input_len

293

294 def _get_verbose_segments(

@@ -409,7 +410,7 @@ class OpenAISpeechToText(OpenAIServing):

410

411 lora_request = self._maybe_get_adapters(request)

412

412 - engine_prompts, duration_s = await self._preprocess_speech_to_text(

413 + engine_inputs, duration_s = await self._preprocess_speech_to_text(

414 request=request,

415 audio_data=audio_data,

416 request_id=request_id,

@@ -420,7 +421,7 @@ class OpenAISpeechToText(OpenAIServing):

421 list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None

422

423 input_len = (

423 - OpenAISpeechToText._get_decoder_prompt_len(engine_prompts)

424 + OpenAISpeechToText._get_decoder_prompt_len(engine_inputs)

425 if request.use_beam_search

426 else 0

427 )

@@ -450,12 +451,12 @@ class OpenAISpeechToText(OpenAIServing):

451 sampling_params.logprobs = 1

452

453 list_result_generator = []

453 - for i, engine_prompt in enumerate(engine_prompts):

454 + for i, engine_input in enumerate(engine_inputs):

455 request_id_item = f"{request_id}_{i}"

456

457 self._log_inputs(

458 request_id_item,

458 - engine_prompt,

459 + engine_input,

460 params=sampling_params,

461 lora_request=lora_request,

462 )

@@ -468,7 +469,7 @@ class OpenAISpeechToText(OpenAIServing):

469

470 if isinstance(sampling_params, BeamSearchParams):

471 generator = self.beam_search(

471 - prompt=engine_prompt,

472 + prompt=engine_input,

473 params=sampling_params,

474 request_id=request_id_item,

475 lora_request=lora_request,

@@ -476,7 +477,7 @@ class OpenAISpeechToText(OpenAIServing):

477 )

478 else:

479 generator = self.engine_client.generate(

479 - engine_prompt,

480 + engine_input,

481 sampling_params,

482 request_id_item,

483 lora_request=lora_request,

vllm/entrypoints/pooling/base/io_processor.py

+10 -10

@@ -18,7 +18,7 @@ from vllm.entrypoints.pooling.typing import (

18 PoolingCompletionLikeRequest,

19 PoolingServeContext,

20 )

21 -from vllm.inputs.data import ProcessorInputs, SingletonPrompt

21 +from vllm.inputs import EngineInput, SingletonPrompt

22 from vllm.renderers import BaseRenderer, merge_kwargs

23 from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq

24 from vllm.tool_parsers import ToolParser

@@ -60,7 +60,7 @@ class PoolingIOProcessor:

60 chat_template_kwargs=request.chat_template_kwargs,

61 trust_request_chat_template=self.trust_request_chat_template,

62 )

63 - _, engine_prompts = self._preprocess_chat_online(

63 + _, engine_inputs = self._preprocess_chat_online(

64 request,

65 request.messages,

66 default_template=self.chat_template,

@@ -68,7 +68,7 @@ class PoolingIOProcessor:

68 default_template_kwargs=None,

69 )

70 elif isinstance(request, PoolingCompletionLikeRequest):

71 - engine_prompts = self._preprocess_completion_online(

71 + engine_inputs = self._preprocess_completion_online(

72 request,

73 prompt_input=request.input,

74 prompt_embeds=None,

@@ -76,7 +76,7 @@ class PoolingIOProcessor:

76 else:

77 raise ValueError(f"Invalid {self.name} request type")

78

79 - ctx.engine_prompts = engine_prompts

79 + ctx.engine_inputs = engine_inputs

80

81 async def pre_process_online_async(self, ctx: PoolingServeContext):

82 self.pre_process_online(ctx)

@@ -100,7 +100,7 @@ class PoolingIOProcessor:

100 self,

101 prompts: PromptType | Sequence[PromptType],

102 tokenization_kwargs: dict[str, Any] | None = None,

103 - ) -> Sequence[ProcessorInputs]:

103 + ) -> Sequence[EngineInput]:

104 return self._preprocess_completion_offline(

105 prompts=prompts, tokenization_kwargs=tokenization_kwargs

106 )

@@ -128,7 +128,7 @@ class PoolingIOProcessor:

128 request: RendererRequest,

129 prompt_input: str | list[str] | list[int] | list[list[int]] | None,

130 prompt_embeds: bytes | list[bytes] | None,

131 - ) -> list[ProcessorInputs]:

131 + ) -> list[EngineInput]:

132 renderer = self.renderer

133 model_config = self.model_config

134

@@ -167,7 +167,7 @@ class PoolingIOProcessor:

167 default_template_kwargs: dict[str, Any] | None,

168 tool_dicts: list[dict[str, Any]] | None = None,

169 tool_parser: type[ToolParser] | None = None,

170 - ) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:

170 + ) -> tuple[list[ConversationMessage], list[EngineInput]]:

171 renderer = self.renderer

172

173 default_template_kwargs = merge_kwargs(

@@ -188,7 +188,7 @@ class PoolingIOProcessor:

188 default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),

189 )

190

191 - (conversation,), (engine_prompt,) = renderer.render_chat(

191 + (conversation,), (engine_input,) = renderer.render_chat(

192 [messages],

193 chat_params,

194 tok_params,

@@ -199,13 +199,13 @@ class PoolingIOProcessor:

199 },

200 )

201

202 - return conversation, [engine_prompt]

202 + return conversation, [engine_input]

203

204 def _preprocess_completion_offline(

205 self,

206 prompts: PromptType | Sequence[PromptType],

207 tokenization_kwargs: dict[str, Any] | None = None,

208 - ) -> Sequence[ProcessorInputs]:

208 + ) -> Sequence[EngineInput]:

209 renderer = self.renderer

210 model_config = self.model_config

211

vllm/entrypoints/pooling/base/serving.py

+8 -8

@@ -20,7 +20,7 @@ from vllm.entrypoints.openai.engine.protocol import ErrorResponse

20 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

21 from vllm.entrypoints.pooling.typing import AnyPoolingRequest, PoolingServeContext

22 from vllm.exceptions import VLLMNotFoundError

23 -from vllm.inputs.data import ProcessorInputs

23 +from vllm.inputs import EngineInput

24 from vllm.lora.request import LoRARequest

25 from vllm.renderers.base import BaseRenderer

26 from vllm.renderers.inputs.preprocess import extract_prompt_components

@@ -106,7 +106,7 @@ class PoolingServing:

106 self,

107 ctx: PoolingServeContext,

108 ):

109 - if ctx.engine_prompts is None:

109 + if ctx.engine_inputs is None:

110 raise ValueError("Engine prompts not available")

111

112 generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

@@ -120,7 +120,7 @@ class PoolingServing:

120 pooling_params = self.io_processor.create_pooling_params(ctx.request)

121 pooling_params.verify(self.model_config)

122

123 - for i, engine_prompt in enumerate(ctx.engine_prompts):

123 + for i, engine_input in enumerate(ctx.engine_inputs):

124 prompt_request_id = (

125 f"{ctx.request_id}-{i}"

126 if ctx.prompt_request_ids is None

@@ -129,13 +129,13 @@ class PoolingServing:

129

130 self._log_inputs(

131 prompt_request_id,

132 - engine_prompt,

132 + engine_input,

133 params=pooling_params,

134 lora_request=ctx.lora_request,

135 )

136

137 generator = self.engine_client.encode(

138 - engine_prompt,

138 + engine_input,

139 pooling_params,

140 prompt_request_id,

141 lora_request=ctx.lora_request,

@@ -151,13 +151,13 @@ class PoolingServing:

151 self,

152 ctx: PoolingServeContext,

153 ):

154 - if ctx.engine_prompts is None:

154 + if ctx.engine_inputs is None:

155 raise ValueError("Engine prompts not available")

156

157 if ctx.result_generator is None:

158 raise ValueError("Result generator not available")

159

160 - num_inputs = len(ctx.engine_prompts)

160 + num_inputs = len(ctx.engine_inputs)

161 final_res_batch: list[PoolingRequestOutput | None]

162 final_res_batch = [None] * num_inputs

163

@@ -317,7 +317,7 @@ class PoolingServing:

317 def _log_inputs(

318 self,

319 request_id: str,

320 - inputs: ProcessorInputs,

320 + inputs: EngineInput,

321 params: PoolingParams,

322 lora_request: LoRARequest | None,

323 ) -> None:

vllm/entrypoints/pooling/embed/io_processor.py

+20 -20

@@ -24,7 +24,7 @@ from vllm.entrypoints.pooling.embed.protocol import (

24 EmbeddingCompletionRequest,

25 )

26 from vllm.entrypoints.pooling.typing import PoolingServeContext

27 -from vllm.inputs.data import ProcessorInputs, token_inputs

27 +from vllm.inputs import EngineInput, tokens_input

28 from vllm.logger import init_logger

29 from vllm.outputs import PoolingOutput, PoolingRequestOutput

30 from vllm.renderers import merge_kwargs

@@ -83,20 +83,20 @@ class EmbedIOProcessor(PoolingIOProcessor):

83 #################################################################

84

85 def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:

86 - if ctx.engine_prompts is None:

86 + if ctx.engine_inputs is None:

87 raise ValueError("Engine prompts not available")

88

89 - ctx.intermediates = ctx.engine_prompts

89 + ctx.intermediates = ctx.engine_inputs

90 request_id = ctx.request_id

91 max_model_len = self.model_config.max_model_len

92 - chunked_engine_prompts: list[ProcessorInputs] = []

92 + chunked_engine_inputs: list[EngineInput] = []

93 prompt_request_ids: list[str] = []

94 - for prompt_idx, engine_prompt in enumerate(ctx.engine_prompts):

95 - token_ids = engine_prompt.get("prompt_token_ids", None)

94 + for prompt_idx, engine_input in enumerate(ctx.engine_inputs):

95 + token_ids = engine_input.get("prompt_token_ids", None)

96 if token_ids is None:

97 raise NotImplementedError(

98 "Long Text Embedding with Chunked Processing does "

99 - "not support EmbedsPrompt and EncoderDecoderInputs."

99 + "not support EmbedsPrompt and EncoderDecoderInput."

100 )

101

102 prompt_token_ids = cast(list[int], token_ids)

@@ -104,14 +104,14 @@ class EmbedIOProcessor(PoolingIOProcessor):

104 for chunk_idx, chunk_tokens in enumerate(

105 chunk_list(prompt_token_ids, max_model_len)

106 ):

107 - chunked_engine_prompts.append(

108 - token_inputs(prompt_token_ids=chunk_tokens)

107 + chunked_engine_inputs.append(

108 + tokens_input(prompt_token_ids=chunk_tokens)

109 )

110 prompt_request_ids.append(

111 f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"

112 )

113

114 - ctx.engine_prompts = chunked_engine_prompts

114 + ctx.engine_inputs = chunked_engine_inputs

115 ctx.prompt_request_ids = prompt_request_ids

116

117 return None

@@ -184,8 +184,8 @@ class EmbedIOProcessor(PoolingIOProcessor):

184 if ctx.intermediates is None:

185 raise ValueError("Original prompts inputs not available")

186

187 - original_engine_prompts = cast(list[ProcessorInputs], ctx.intermediates)

188 - num_prompts = len(original_engine_prompts)

187 + original_engine_inputs = cast(list[EngineInput], ctx.intermediates)

188 + num_prompts = len(original_engine_inputs)

189

190 # Finalize aggregated results

191 final_res_batch: list[PoolingRequestOutput] = []

@@ -211,12 +211,12 @@ class EmbedIOProcessor(PoolingIOProcessor):

211 pooling_output_data = PoolingOutput(data=final_embedding)

212

213 # Get original prompt token IDs for this prompt

214 - original_prompt = original_engine_prompts[prompt_idx]

214 + original_prompt = original_engine_inputs[prompt_idx]

215 token_ids = original_prompt.get("prompt_token_ids", None)

216 if token_ids is None:

217 raise NotImplementedError(

218 "Long Text Embedding with Chunked Processing does "

219 - "not support EmbedsPrompt and EncoderDecoderInputs."

219 + "not support EmbedsPrompt and EncoderDecoderInput."

220 )

221

222 original_token_ids = cast(list[int], token_ids)

@@ -372,7 +372,7 @@ class EmbedIOProcessor(PoolingIOProcessor):

372 ]

373 for uri in request.images

374 ]

375 - ctx.engine_prompts = self._batch_render_chat(

375 + ctx.engine_inputs = self._batch_render_chat(

376 request, all_messages, truncate_prompt_tokens, truncation_side

377 )

378

@@ -382,7 +382,7 @@ class EmbedIOProcessor(PoolingIOProcessor):

382 self._mixed_input_to_messages(inp, task_prefix=task_prefix)

383 for inp in request.inputs

384 ]

385 - ctx.engine_prompts = self._batch_render_chat(

385 + ctx.engine_inputs = self._batch_render_chat(

386 request, all_messages, truncate_prompt_tokens, truncation_side

387 )

388

@@ -396,7 +396,7 @@ class EmbedIOProcessor(PoolingIOProcessor):

396 truncate_prompt_tokens=truncate_prompt_tokens,

397 truncation_side=truncation_side,

398 )

399 - ctx.engine_prompts = self._preprocess_completion_online(

399 + ctx.engine_inputs = self._preprocess_completion_online(

400 proxy, prompt_input=proxy.input, prompt_embeds=None

401 )

402

@@ -406,7 +406,7 @@ class EmbedIOProcessor(PoolingIOProcessor):

406 all_messages: Sequence[list[ChatCompletionMessageParam]],

407 truncate_prompt_tokens: int | None,

408 truncation_side: Literal["left", "right"] | None,

409 - ) -> list[ProcessorInputs]:

409 + ) -> list[EngineInput]:

410 """Batch-render multiple conversations through the chat template."""

411 if not all_messages:

412 return []

@@ -438,8 +438,8 @@ class EmbedIOProcessor(PoolingIOProcessor):

438 default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),

439 )

440

441 - _, engine_prompts = renderer.render_chat(all_messages, chat_params, tok_params)

442 - return engine_prompts

441 + _, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)

442 + return engine_inputs

443

444 def _validate_input_type(self, input_type: str | None) -> None:

445 """Raise if *input_type* is not supported by this model."""

vllm/entrypoints/pooling/pooling/serving.py

+9 -9

@@ -33,7 +33,7 @@ from vllm.entrypoints.pooling.utils import (

33 encode_pooling_output_float,

34 )

35 from vllm.entrypoints.serve.render.serving import OpenAIServingRender

36 -from vllm.inputs import ProcessorInputs

36 +from vllm.inputs import EngineInput

37 from vllm.logger import init_logger

38 from vllm.outputs import PoolingRequestOutput

39 from vllm.renderers.inputs.preprocess import prompt_to_seq

@@ -110,7 +110,7 @@ class OpenAIServingPooling(OpenAIServing):

110 request.task,

111 )

112

113 - engine_prompts: Sequence[ProcessorInputs]

113 + engine_inputs: Sequence[EngineInput]

114 if use_io_processor := isinstance(request, IOProcessorRequest):

115 if self.io_processor is None:

116 raise ValueError(

@@ -125,7 +125,7 @@ class OpenAIServingPooling(OpenAIServing):

125 raw_prompts = await self.io_processor.pre_process_async(

126 prompt=validated_prompt, request_id=request_id

127 )

128 - engine_prompts = await self.openai_serving_render.preprocess_cmpl(

128 + engine_inputs = await self.openai_serving_render.preprocess_cmpl(

129 request,

130 prompt_to_seq(raw_prompts),

131 )

@@ -138,7 +138,7 @@ class OpenAIServingPooling(OpenAIServing):

138 if error_check_ret is not None:

139 return error_check_ret

140

141 - _, engine_prompts = await self.openai_serving_render.preprocess_chat(

141 + _, engine_inputs = await self.openai_serving_render.preprocess_chat(

142 request,

143 request.messages,

144 default_template=self.chat_template,

@@ -146,7 +146,7 @@ class OpenAIServingPooling(OpenAIServing):

146 default_template_kwargs=None,

147 )

148 elif isinstance(request, PoolingCompletionRequest):

149 - engine_prompts = await self.openai_serving_render.preprocess_completion(

149 + engine_inputs = await self.openai_serving_render.preprocess_completion(

150 request,

151 prompt_input=request.input,

152 prompt_embeds=None,

@@ -165,12 +165,12 @@ class OpenAIServingPooling(OpenAIServing):

165 else:

166 pooling_params = request.to_pooling_params() # type: ignore

167

168 - for i, engine_prompt in enumerate(engine_prompts):

168 + for i, engine_input in enumerate(engine_inputs):

169 request_id_item = f"{request_id}-{i}"

170

171 self._log_inputs(

172 request_id_item,

173 - engine_prompt,

173 + engine_input,

174 params=pooling_params,

175 lora_request=lora_request,

176 )

@@ -182,7 +182,7 @@ class OpenAIServingPooling(OpenAIServing):

182 )

183

184 generator = self.engine_client.encode(

185 - engine_prompt,

185 + engine_input,

186 pooling_params,

187 request_id_item,

188 lora_request=lora_request,

@@ -221,7 +221,7 @@ class OpenAIServingPooling(OpenAIServing):

221 return IOProcessorResponse(request_id=request_id, data=output)

222

223 assert isinstance(request, (PoolingCompletionRequest, PoolingChatRequest))

224 - num_prompts = len(engine_prompts)

224 + num_prompts = len(engine_inputs)

225

226 # Non-streaming response

227 final_res_batch: list[PoolingRequestOutput | None]

vllm/entrypoints/pooling/score/serving.py

+36 -45

@@ -35,7 +35,7 @@ from vllm.entrypoints.pooling.score.utils import (

35 parse_score_data_single,

36 validate_score_input,

37 )

38 -from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs

38 +from vllm.inputs import EngineInput, TokensPrompt, tokens_input

39 from vllm.logger import init_logger

40 from vllm.lora.request import LoRARequest

41 from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput

@@ -110,12 +110,12 @@ class ServingScores(OpenAIServing):

110 *(encode_async(t, **tokenization_kwargs) for t in input_texts)

111 )

112

113 - engine_prompts: list[ProcessorInputs] = []

113 + engine_inputs: list[EngineInput] = []

114 for tok_result, input_text in zip(tokenized_prompts, input_texts):

115 text_token_prompt = self._validate_input(request, tok_result, input_text)

116

117 - engine_prompts.append(

118 - token_inputs(

117 + engine_inputs.append(

118 + tokens_input(

119 text_token_prompt["prompt_token_ids"],

120 prompt=input_text,

121 )

@@ -125,19 +125,19 @@ class ServingScores(OpenAIServing):

125 generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

126 pooling_params = request.to_pooling_params("embed")

127

128 - for i, engine_prompt in enumerate(engine_prompts):

128 + for i, engine_input in enumerate(engine_inputs):

129 request_id_item = f"{request_id}-{i}"

130

131 self._log_inputs(

132 request_id_item,

133 - engine_prompt,

133 + engine_input,

134 params=pooling_params,

135 lora_request=lora_request,

136 )

137

138 generators.append(

139 self.engine_client.encode(

140 - engine_prompt,

140 + engine_input,

141 pooling_params,

142 request_id_item,

143 lora_request=lora_request,

@@ -151,7 +151,7 @@ class ServingScores(OpenAIServing):

151 # Non-streaming response

152 final_res_batch: list[PoolingRequestOutput] = []

153

154 - embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_prompts)

154 + embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_inputs)

155

156 async for i, res in result_generator:

157 embeddings[i] = res

@@ -183,7 +183,7 @@ class ServingScores(OpenAIServing):

183 request: RerankRequest | ScoreRequest,

184 tokenizer: TokenizerLike,

185 tokenization_kwargs: dict[str, Any],

186 - ) -> tuple[str, TokensPrompt]:

186 + ) -> TokensPrompt:

187 """Parse a single ScoreData into a text + optional multimodal

188 TokensPrompt for late-interaction encoding.

189

@@ -197,21 +197,22 @@ class ServingScores(OpenAIServing):

197 else:

198 text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)

199

200 - prompt_inputs = tokenizer(text, **tokenization_kwargs)

201 - self._validate_input(request, prompt_inputs["input_ids"], text)

200 + prompt_ids = tokenizer.encode(text, **tokenization_kwargs)

201 + self._validate_input(request, prompt_ids, text)

202

203 - engine_prompt = TokensPrompt(

204 - prompt_token_ids=prompt_inputs["input_ids"],

203 + tok_prompt = TokensPrompt(

204 + prompt_token_ids=prompt_ids,

205 + prompt=text,

206 )

207

208 if mm_data is not None:

208 - engine_prompt["multi_modal_data"] = mm_data

209 + tok_prompt["multi_modal_data"] = mm_data

210 if mm_uuids is not None:

210 - engine_prompt["multi_modal_uuids"] = mm_uuids

211 + tok_prompt["multi_modal_uuids"] = mm_uuids

212 if request.mm_processor_kwargs is not None:

212 - engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs

213 + tok_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs

214

214 - return text, engine_prompt

215 + return tok_prompt

216

217 async def _late_interaction_score(

218 self,

@@ -240,7 +241,7 @@ class ServingScores(OpenAIServing):

241 executor=self._tokenizer_executor,

242 )

243

243 - preprocessed = await asyncio.gather(

244 + tok_prompts = await asyncio.gather(

245 *(

246 preprocess_async(

247 data=d,

@@ -253,12 +254,8 @@ class ServingScores(OpenAIServing):

254 )

255 )

256

256 - query_prompts: list[TokensPrompt] = [

257 - prompt for _, prompt in preprocessed[: len(data_1)]

258 - ]

259 - doc_prompts: list[TokensPrompt] = [

260 - prompt for _, prompt in preprocessed[len(data_1) :]

261 - ]

257 + query_prompts = tok_prompts[: len(data_1)]

258 + doc_prompts = tok_prompts[len(data_1) :]

259

260 default_pooling_params = request.to_pooling_params("token_embed")

261

@@ -268,7 +265,7 @@ class ServingScores(OpenAIServing):

265 query_prompts

266 )

267 query_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

271 - for i, engine_prompt in enumerate(query_prompts):

268 + for i, tok_prompt in enumerate(query_prompts):

269 request_id_item = f"{request_id}-query-{i}"

270 pooling_params = default_pooling_params.clone()

271 pooling_params.late_interaction_params = (

@@ -280,14 +277,14 @@ class ServingScores(OpenAIServing):

277

278 self._log_inputs(

279 request_id_item,

283 - engine_prompt,

280 + tok_prompt,

281 params=pooling_params,

282 lora_request=lora_request,

283 )

284

285 query_generators.append(

286 self.engine_client.encode(

290 - engine_prompt,

287 + tok_prompt,

288 pooling_params,

289 request_id_item,

290 lora_request=lora_request,

@@ -306,7 +303,7 @@ class ServingScores(OpenAIServing):

303

304 # stage 2: encode docs and return scalar scores from workers.

305 doc_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

309 - for i, engine_prompt in enumerate(doc_prompts):

306 + for i, tok_prompt in enumerate(doc_prompts):

307 request_id_item = f"{request_id}-doc-{i}"

308 query_idx = 0 if len(query_prompts) == 1 else i

309 pooling_params = default_pooling_params.clone()

@@ -316,14 +313,14 @@ class ServingScores(OpenAIServing):

313

314 self._log_inputs(

315 request_id_item,

319 - engine_prompt,

316 + tok_prompt,

317 params=pooling_params,

318 lora_request=lora_request,

319 )

320

321 doc_generators.append(

322 self.engine_client.encode(

326 - engine_prompt,

323 + tok_prompt,

324 pooling_params,

325 request_id_item,

326 lora_request=lora_request,

@@ -404,28 +401,22 @@ class ServingScores(OpenAIServing):

401 )

402 )

403

407 - request_prompts: list[str] = []

408 - engine_prompts: list[TokensPrompt] = []

409 - for full_prompt, engine_prompt in preprocessed_prompts:

410 - request_prompts.append(full_prompt)

411 - engine_prompts.append(engine_prompt)

412 -

404 # Schedule the request and get the result generator.

405 generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

406

407 default_pooling_params = request.to_pooling_params("classify")

408

418 - for i, engine_prompt in enumerate(engine_prompts):

409 + for i, (full_prompt, tok_prompt) in enumerate(preprocessed_prompts):

410 request_id_item = f"{request_id}-{i}"

411

412 self._log_inputs(

413 request_id_item,

423 - request_prompts[i],

414 + full_prompt,

415 params=default_pooling_params,

416 lora_request=lora_request,

417 )

418

428 - if token_type_ids := engine_prompt.pop("token_type_ids", None):

419 + if token_type_ids := tok_prompt.pop("token_type_ids", None):

420 pooling_params = default_pooling_params.clone()

421 compressed = compress_token_type_ids(token_type_ids)

422 pooling_params.extra_kwargs = {"compressed_token_type_ids": compressed}

@@ -433,7 +424,7 @@ class ServingScores(OpenAIServing):

424 pooling_params = default_pooling_params

425

426 generator = self.engine_client.encode(

436 - engine_prompt,

427 + tok_prompt,

428 pooling_params,

429 request_id_item,

430 lora_request=lora_request,

@@ -447,7 +438,7 @@ class ServingScores(OpenAIServing):

438

439 # Non-streaming response

440 final_res_batch: list[PoolingRequestOutput | None] = [None] * len(

450 - engine_prompts

441 + preprocessed_prompts

442 )

443

444 async for i, res in result_generator:

@@ -464,7 +455,7 @@ class ServingScores(OpenAIServing):

455 data_2: ScoreData,

456 ) -> tuple[str, TokensPrompt]:

457 model_config = self.model_config

467 - full_prompt, engine_prompt = get_score_prompt(

458 + full_prompt, engine_input = get_score_prompt(

459 model_config=model_config,

460 data_1=data_1,

461 data_2=data_2,

@@ -472,11 +463,11 @@ class ServingScores(OpenAIServing):

463 tokenization_kwargs=tokenization_kwargs,

464 score_template=self.score_template,

465 )

475 - self._validate_input(request, engine_prompt["prompt_token_ids"], full_prompt)

466 + self._validate_input(request, engine_input["prompt_token_ids"], full_prompt)

467 if request.mm_processor_kwargs is not None:

477 - engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs

468 + engine_input["mm_processor_kwargs"] = request.mm_processor_kwargs

469

479 - return full_prompt, engine_prompt

470 + return full_prompt, engine_input

471

472 async def _run_scoring(

473 self,

vllm/entrypoints/pooling/score/utils.py

+7 -3

@@ -20,10 +20,14 @@ from vllm.entrypoints.chat_utils import (

20 MultiModalItemTracker,

21 _parse_chat_message_content_parts,

22 )

23 -from vllm.inputs import TokensPrompt

24 -from vllm.inputs.data import PromptType, TextPrompt

23 +from vllm.inputs import (

24 + MultiModalDataDict,

25 + MultiModalUUIDDict,

26 + PromptType,

27 + TextPrompt,

28 + TokensPrompt,

29 +)

30 from vllm.model_executor.models.interfaces import supports_score_template

26 -from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict

31 from vllm.outputs import PoolingRequestOutput

32 from vllm.renderers.hf import safe_apply_chat_template

33 from vllm.tokenizers import TokenizerLike

vllm/entrypoints/pooling/typing.py

+2 -2

@@ -32,7 +32,7 @@ from vllm.entrypoints.pooling.score.protocol import (

32 ScoreRequest,

33 ScoreResponse,

34 )

35 -from vllm.inputs import ProcessorInputs

35 +from vllm.inputs import EngineInput

36 from vllm.lora.request import LoRARequest

37

38 PoolingCompletionLikeRequest: TypeAlias = (

@@ -74,7 +74,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):

74 created_time: int = field(default_factory=lambda: int(time.time()))

75 lora_request: LoRARequest | None = None

76

77 - engine_prompts: list[ProcessorInputs] | None = None

77 + engine_inputs: list[EngineInput] | None = None

78 prompt_request_ids: list[str] | None = None

79 intermediates: Any | None = None

80

vllm/entrypoints/serve/disagg/protocol.py

+9 -8

@@ -33,19 +33,20 @@ class MultiModalFeatures(BaseModel):

33 """Lightweight multimodal metadata produced by the render step.

34

35 Carries hashes (for cache lookup / identification) and placeholder

36 - positions so the downstream ``/generate`` service knows *where* in

36 + positions so the downstream `/generate` service knows *where* in

37 the token sequence each multimodal item lives.

38

39 - .. note:: Phase 1 — metadata only.

40 - Phase 2 should add ``mm_kwargs`` (processed tensor data) using a

41 - binary transport so the ``/generate`` side can skip re-processing.

42 - The ``/generate`` endpoint must also be updated to inject these

43 - features into ``ProcessorInputs`` before passing to

44 - ``InputProcessor.process_inputs``.

39 + Note:

40 + Phase 1 — metadata only.

41 + Phase 2 should add `mm_kwargs` (processed tensor data) using a

42 + binary transport so the ``/generate` side can skip re-processing.

43 + The `/generate` endpoint must also be updated to inject these

44 + features into `EngineInput` before passing to

45 + `InputProcessor.process_inputs`.

46 """

47

48 mm_hashes: dict[str, list[str]]

48 - """Per-modality item hashes, e.g. ``{"image": ["abc", "def"]}``."""

49 + """Per-modality item hashes, e.g. `{"image": ["abc", "def"]}`."""

50

51 mm_placeholders: dict[str, list[PlaceholderRangeInfo]]

52 """Per-modality placeholder ranges in the token sequence."""

vllm/entrypoints/serve/disagg/serving.py

+3 -5

@@ -99,13 +99,11 @@ class ServingTokens(OpenAIServing):

99 if raw_request:

100 raw_request.state.request_metadata = request_metadata

101

102 - engine_prompts = await self.openai_serving_render.preprocess_completion(

102 + (engine_input,) = await self.openai_serving_render.preprocess_completion(

103 request,

104 prompt_input=request.token_ids,

105 prompt_embeds=None,

106 )

107 - assert len(engine_prompts) == 1

108 - engine_prompt = engine_prompts[0]

107

108 # Schedule the request and get the result generator.

109 result_generator: AsyncGenerator[RequestOutput, None] | None = None

@@ -115,7 +113,7 @@ class ServingTokens(OpenAIServing):

113

114 self._log_inputs(

115 request_id,

118 - engine_prompt,

116 + engine_input,

117 params=sampling_params,

118 lora_request=lora_request,

119 )

@@ -127,7 +125,7 @@ class ServingTokens(OpenAIServing):

125 )

126

127 result_generator = self.engine_client.generate(

130 - engine_prompt,

128 + engine_input,

129 sampling_params,

130 request_id,

131 lora_request=lora_request,

vllm/entrypoints/serve/render/serving.py

+38 -37

@@ -34,9 +34,15 @@ from vllm.entrypoints.utils import (

34 create_error_response,

35 get_max_tokens,

36 )

37 -from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt

37 +from vllm.inputs import (

38 + EngineInput,

39 + MultiModalHashes,

40 + MultiModalPlaceholders,

41 + PromptType,

42 + SingletonPrompt,

43 + tokens_input,

44 +)

45 from vllm.logger import init_logger

39 -from vllm.multimodal.inputs import MultiModalHashes, MultiModalPlaceholderDict

46 from vllm.parser import ParserManager

47 from vllm.renderers import BaseRenderer, merge_kwargs

48 from vllm.renderers.inputs.preprocess import (

@@ -127,22 +133,22 @@ class OpenAIServingRender:

133 if isinstance(result, ErrorResponse):

134 return result

135

130 - _, engine_prompts = result

136 + _, engine_inputs = result

137

132 - if len(engine_prompts) != 1:

138 + if len(engine_inputs) != 1:

139 return self.create_error_response(

134 - f"Expected exactly 1 engine prompt, got {len(engine_prompts)}"

140 + f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"

141 )

142

137 - engine_prompt = engine_prompts[0]

143 + engine_input = engine_inputs[0]

144

139 - prompt_components = extract_prompt_components(self.model_config, engine_prompt)

145 + prompt_components = extract_prompt_components(self.model_config, engine_input)

146 token_ids = prompt_components.token_ids

147 if not token_ids:

148 return self.create_error_response("No token_ids rendered")

149 token_ids = list(token_ids)

150

145 - input_length = extract_prompt_len(self.model_config, engine_prompt)

151 + input_length = extract_prompt_len(self.model_config, engine_input)

152 max_tokens = get_max_tokens(

153 self.model_config.max_model_len,

154 request.max_completion_tokens

@@ -159,7 +165,7 @@ class OpenAIServingRender:

165 return GenerateRequest(

166 request_id=request_id,

167 token_ids=token_ids,

162 - features=self._extract_mm_features(engine_prompt),

168 + features=self._extract_mm_features(engine_input),

169 sampling_params=params,

170 model=request.model,

171 stream=bool(request.stream),

@@ -171,7 +177,7 @@ class OpenAIServingRender:

177 async def render_chat(

178 self,

179 request: ChatCompletionRequest,

174 - ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:

180 + ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:

181 """Core preprocessing logic for chat requests (no model/engine check).

182

183 Called directly by render_chat_request and delegated to by

@@ -184,7 +190,6 @@ class OpenAIServingRender:

190 if is_mistral_tokenizer(tokenizer):

191 # because of issues with pydantic we need to potentially

192 # re-serialize the tool_calls field of the request

187 - # for more info: see comment in `maybe_serialize_tool_calls`

193 _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]

194 _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]

195 _mt.validate_request_params(request)

@@ -232,7 +237,7 @@ class OpenAIServingRender:

237 if error_check_ret is not None:

238 return error_check_ret

239

235 - conversation, engine_prompts = await self.preprocess_chat(

240 + conversation, engine_inputs = await self.preprocess_chat(

241 request,

242 request.messages,

243 default_template=self.chat_template,

@@ -244,11 +249,11 @@ class OpenAIServingRender:

249 else:

250 # For GPT-OSS.

251 should_include_tools = tool_dicts is not None

247 - conversation, engine_prompts = self._make_request_with_harmony(

252 + conversation, engine_inputs = self._make_request_with_harmony(

253 request, should_include_tools

254 )

255

251 - return conversation, engine_prompts

256 + return conversation, engine_inputs

257

258 async def render_completion_request(

259 self,

@@ -266,16 +271,16 @@ class OpenAIServingRender:

271 if isinstance(result, ErrorResponse):

272 return result

273 generate_requests: list[GenerateRequest] = []

269 - for engine_prompt in result:

274 + for engine_input in result:

275 prompt_components = extract_prompt_components(

271 - self.model_config, engine_prompt

276 + self.model_config, engine_input

277 )

278 token_ids = prompt_components.token_ids

279 if not token_ids:

280 return self.create_error_response("No token_ids rendered")

281 token_ids = list(token_ids)

282

278 - input_length = extract_prompt_len(self.model_config, engine_prompt)

283 + input_length = extract_prompt_len(self.model_config, engine_input)

284 max_tokens = get_max_tokens(

285 self.model_config.max_model_len,

286 request.max_tokens,

@@ -293,7 +298,7 @@ class OpenAIServingRender:

298 GenerateRequest(

299 request_id=request_id,

300 token_ids=token_ids,

296 - features=self._extract_mm_features(engine_prompt),

301 + features=self._extract_mm_features(engine_input),

302 sampling_params=params,

303 model=request.model,

304 stream=bool(request.stream),

@@ -308,7 +313,7 @@ class OpenAIServingRender:

313 async def render_completion(

314 self,

315 request: CompletionRequest,

311 - ) -> list[ProcessorInputs] | ErrorResponse:

316 + ) -> list[EngineInput] | ErrorResponse:

317 """Core preprocessing logic for completion requests (no model/engine check).

318

319 Called directly by render_completion_request and delegated to by

@@ -326,28 +331,28 @@ class OpenAIServingRender:

331 "prompt_logprobs is not compatible with prompt embeds."

332 )

333

329 - engine_prompts = await self.preprocess_completion(

334 + engine_inputs = await self.preprocess_completion(

335 request,

336 prompt_input=request.prompt,

337 prompt_embeds=request.prompt_embeds,

338 )

339

335 - return engine_prompts

340 + return engine_inputs

341

342 @staticmethod

343 def _extract_mm_features(

339 - engine_prompt: ProcessorInputs,

344 + engine_input: EngineInput,

345 ) -> MultiModalFeatures | None:

346 """Extract multimodal metadata from a rendered engine prompt.

347

348 Returns ``None`` for text-only prompts.

349 """

345 - if engine_prompt.get("type") != "multimodal":

350 + if engine_input.get("type") != "multimodal":

351 return None

352

348 - # At this point engine_prompt is a MultiModalInputs TypedDict.

349 - mm_hashes: MultiModalHashes = engine_prompt["mm_hashes"] # type: ignore[typeddict-item]

350 - raw_placeholders: MultiModalPlaceholderDict = engine_prompt["mm_placeholders"] # type: ignore[typeddict-item]

353 + # At this point engine_input is a MultiModalInputs TypedDict.

354 + mm_hashes: MultiModalHashes = engine_input["mm_hashes"] # type: ignore[typeddict-item]

355 + raw_placeholders: MultiModalPlaceholders = engine_input["mm_placeholders"] # type: ignore[typeddict-item]

356

357 mm_placeholders = {

358 modality: [

@@ -401,13 +406,9 @@ class OpenAIServingRender:

406

407 # Render prompt token ids.

408 prompt_token_ids = render_for_completion(messages)

404 - engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)

405 -

406 - # Add cache_salt if provided in the request

407 - if request.cache_salt is not None:

408 - engine_prompt["cache_salt"] = request.cache_salt

409 + engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)

410

410 - return messages, [engine_prompt]

411 + return messages, [engine_input]

412

413 def create_error_response(

414 self,

@@ -450,7 +451,7 @@ class OpenAIServingRender:

451 request: Any,

452 prompt_input: str | list[str] | list[int] | list[list[int]] | None,

453 prompt_embeds: bytes | list[bytes] | None,

453 - ) -> list[ProcessorInputs]:

454 + ) -> list[EngineInput]:

455 """Copied from OpenAIServing._preprocess_completion."""

456 prompts = list[SingletonPrompt | bytes]()

457 if prompt_embeds is not None: # embeds take higher priority

@@ -463,7 +464,7 @@ class OpenAIServingRender:

464 self,

465 request: Any,

466 prompts: Sequence[PromptType | bytes],

466 - ) -> list[ProcessorInputs]:

467 + ) -> list[EngineInput]:

468 """Copied from OpenAIServing._preprocess_cmpl."""

469 renderer = self.renderer

470 model_config = self.model_config

@@ -497,7 +498,7 @@ class OpenAIServingRender:

498 default_template_kwargs: dict[str, Any] | None,

499 tool_dicts: list[dict[str, Any]] | None = None,

500 tool_parser: type[ToolParser] | None = None,

500 - ) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:

501 + ) -> tuple[list[ConversationMessage], list[EngineInput]]:

502 """Copied from OpenAIServing._preprocess_chat."""

503 renderer = self.renderer

504 mm_config = self.model_config.multimodal_config

@@ -519,7 +520,7 @@ class OpenAIServingRender:

520 default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),

521 )

522

522 - (conversation,), (engine_prompt,) = await renderer.render_chat_async(

523 + (conversation,), (engine_input,) = await renderer.render_chat_async(

524 [messages],

525 chat_params,

526 tok_params,

@@ -546,4 +547,4 @@ class OpenAIServingRender:

547 tokenizer = renderer.get_tokenizer()

548 request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore[arg-type]

549

549 - return conversation, [engine_prompt]

550 + return conversation, [engine_input]

vllm/entrypoints/serve/tokenize/serving.py

+9 -9

@@ -20,7 +20,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (

20 TokenizeResponse,

21 TokenizerInfoResponse,

22 )

23 -from vllm.inputs import TokensPrompt, token_inputs

23 +from vllm.inputs import TokensPrompt, tokens_input

24 from vllm.logger import init_logger

25 from vllm.tokenizers import TokenizerLike

26

@@ -79,7 +79,7 @@ class OpenAIServingTokenization(OpenAIServing):

79 if error_check_ret is not None:

80 return error_check_ret

81

82 - _, engine_prompts = await self.openai_serving_render.preprocess_chat(

82 + _, engine_inputs = await self.openai_serving_render.preprocess_chat(

83 request,

84 request.messages,

85 default_template=self.chat_template,

@@ -88,22 +88,22 @@ class OpenAIServingTokenization(OpenAIServing):

88 tool_dicts=tool_dicts,

89 )

90 else:

91 - engine_prompts = await self.openai_serving_render.preprocess_completion(

91 + engine_inputs = await self.openai_serving_render.preprocess_completion(

92 request,

93 prompt_input=request.prompt,

94 prompt_embeds=None,

95 )

96

97 input_ids: list[int] = []

98 - for engine_prompt in engine_prompts:

98 + for engine_input in engine_inputs:

99 self._log_inputs(

100 request_id,

101 - engine_prompt,

101 + engine_input,

102 params=None,

103 lora_request=lora_request,

104 )

105

106 - prompt_components = self._extract_prompt_components(engine_prompt)

106 + prompt_components = self._extract_prompt_components(engine_input)

107 if prompt_components.token_ids is not None:

108 input_ids.extend(prompt_components.token_ids)

109

@@ -134,16 +134,16 @@ class OpenAIServingTokenization(OpenAIServing):

134

135 self._log_inputs(

136 request_id,

137 - token_inputs(request.tokens),

137 + tokens_input(request.tokens),

138 params=None,

139 lora_request=lora_request,

140 )

141

142 - engine_prompt = await self.renderer.tokenize_prompt_async(

142 + tok_prompt = await self.renderer.tokenize_prompt_async(

143 TokensPrompt(prompt_token_ids=request.tokens),

144 request.build_tok_params(self.model_config),

145 )

146 - prompt_text = engine_prompt["prompt"] # type: ignore[typeddict-item]

146 + prompt_text = tok_prompt["prompt"] # type: ignore[typeddict-item]

147

148 return DetokenizeResponse(prompt=prompt_text)

149

vllm/inputs/__init__.py

+43 -17

@@ -1,38 +1,64 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from .data import (

4 +from .engine import (

5 + DecoderOnlyEngineInput,

6 + EmbedsInput,

7 + EncoderDecoderInput,

8 + EngineInput,

9 + MultiModalEncDecInput,

10 + MultiModalHashes,

11 + MultiModalInput,

12 + MultiModalPlaceholders,

13 + SingletonInput,

14 + TokensInput,

15 + build_enc_dec_input,

16 + embeds_input,

17 + mm_enc_dec_input,

18 + mm_input,

19 + split_enc_dec_input,

20 + tokens_input,

21 +)

22 +from .llm import (

23 DataPrompt,

6 - DecoderOnlyInputs,

7 - EmbedsInputs,

24 EmbedsPrompt,

9 - EncoderDecoderInputs,

25 ExplicitEncoderDecoderPrompt,

11 - ProcessorInputs,

26 + ModalityData,

27 + MultiModalDataBuiltins,

28 + MultiModalDataDict,

29 + MultiModalUUIDDict,

30 PromptType,

13 - SingletonInputs,

31 SingletonPrompt,

32 TextPrompt,

16 - TokenInputs,

33 TokensPrompt,

18 - embeds_inputs,

19 - token_inputs,

34 )

35

36 __all__ = [

37 + "ModalityData",

38 + "MultiModalDataBuiltins",

39 + "MultiModalDataDict",

40 + "MultiModalUUIDDict",

41 "DataPrompt",

42 "TextPrompt",

43 "TokensPrompt",

44 "PromptType",

45 "SingletonPrompt",

46 "ExplicitEncoderDecoderPrompt",

29 - "TokenInputs",

30 - "EmbedsInputs",

47 "EmbedsPrompt",

32 - "token_inputs",

33 - "embeds_inputs",

34 - "DecoderOnlyInputs",

35 - "EncoderDecoderInputs",

36 - "ProcessorInputs",

37 - "SingletonInputs",

48 + "MultiModalHashes",

49 + "MultiModalPlaceholders",

50 + "TokensInput",

51 + "EmbedsInput",

52 + "MultiModalInput",

53 + "MultiModalEncDecInput",

54 + "tokens_input",

55 + "embeds_input",

56 + "mm_input",

57 + "mm_enc_dec_input",

58 + "build_enc_dec_input",

59 + "split_enc_dec_input",

60 + "DecoderOnlyEngineInput",

61 + "EncoderDecoderInput",

62 + "SingletonInput",

63 + "EngineInput",

64 ]

vllm/inputs/data.py deleted

-413

@@ -1,413 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from typing import TYPE_CHECKING, Any, Literal, TypeAlias

4 -

5 -import torch

6 -from typing_extensions import NotRequired, TypedDict, assert_never

7 -

8 -if TYPE_CHECKING:

9 - from vllm.multimodal.inputs import (

10 - MultiModalDataDict,

11 - MultiModalEncDecInputs,

12 - MultiModalInputs,

13 - MultiModalUUIDDict,

14 - )

15 -else:

16 - MultiModalDataDict = object

17 - MultiModalEncDecInputs = object

18 - MultiModalInputs = object

19 - MultiModalUUIDDict = object

20 -

21 -

22 -# Inputs to LLM API

23 -class _PromptOptions(TypedDict):

24 - """

25 - Additional options available to all

26 - [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt].

27 - """

28 -

29 - multi_modal_data: NotRequired[MultiModalDataDict | None]

30 - """

31 - Optional multi-modal data to pass to the model,

32 - if the model supports it.

33 - """

34 -

35 - mm_processor_kwargs: NotRequired[dict[str, Any] | None]

36 - """

37 - Optional multi-modal processor kwargs to be forwarded to the

38 - multimodal input mapper & processor. Note that if multiple modalities

39 - have registered mappers etc for the model being considered, we attempt

40 - to pass the mm_processor_kwargs to each of them.

41 - """

42 -

43 - multi_modal_uuids: NotRequired[MultiModalUUIDDict]

44 - """

45 - Optional user-specified UUIDs for multimodal items, mapped by modality.

46 - Lists must match the number of items per modality and may contain `None`.

47 - For `None` entries, the hasher will compute IDs automatically; non-None

48 - entries override the default hashes for caching, and MUST be unique per

49 - multimodal item.

50 - """

51 -

52 - cache_salt: NotRequired[str]

53 - """

54 - Optional cache salt to be used for prefix caching.

55 - """

56 -

57 -

58 -class TextPrompt(_PromptOptions):

59 - """Schema for a text prompt."""

60 -

61 - prompt: str

62 - """The input text to be tokenized before passing to the model."""

63 -

64 -

65 -class TokensPrompt(_PromptOptions):

66 - """Schema for a tokenized prompt."""

67 -

68 - prompt_token_ids: list[int]

69 - """A list of token IDs to pass to the model."""

70 -

71 - prompt: NotRequired[str]

72 - """The prompt text corresponding to the token IDs, if available."""

73 -

74 - token_type_ids: NotRequired[list[int]]

75 - """A list of token type IDs to pass to the cross encoder model."""

76 -

77 -

78 -class EmbedsPrompt(_PromptOptions):

79 - """Schema for a prompt provided via token embeddings."""

80 -

81 - prompt_embeds: torch.Tensor

82 - """The embeddings of the prompt."""

83 -

84 - prompt: NotRequired[str]

85 - """The prompt text corresponding to the token embeddings, if available."""

86 -

87 -

88 -DecoderOnlyPrompt: TypeAlias = (

89 - str | TextPrompt | list[int] | TokensPrompt | EmbedsPrompt

90 -)

91 -"""

92 -Schema of a prompt for a decoder-only model:

93 -

94 -- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])

95 -- A tokenized prompt (list of token IDs, or

96 - [`TokensPrompt`][vllm.inputs.data.TokensPrompt])

97 -- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])

98 -

99 -For encoder-decoder models, passing a singleton prompt is shorthand for passing

100 -`ExplicitEncoderDecoderPrompt(encoder_prompt=prompt, decoder_prompt=None)`.

101 -"""

102 -

103 -

104 -EncoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt

105 -"""

106 -Schema of a prompt for the encoder part of a encoder-decoder model:

107 -

108 -- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])

109 -- A tokenized prompt (list of token IDs, or

110 - [`TokensPrompt`][vllm.inputs.data.TokensPrompt])

111 -"""

112 -

113 -

114 -DecoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt

115 -"""

116 -Schema of a prompt for the decoder part of an encoder-decoder model:

117 -

118 -- A text prompt (string or [`TextPrompt`][vllm.inputs.data.TextPrompt])

119 -- A tokenized prompt (list of token IDs, or

120 - [`TokensPrompt`][vllm.inputs.data.TokensPrompt])

121 -

122 -Note:

123 - Multi-modal inputs are not supported for decoder prompts.

124 -"""

125 -

126 -

127 -class ExplicitEncoderDecoderPrompt(TypedDict):

128 - """

129 - Schema for a pair of encoder and decoder singleton prompts.

130 -

131 - Note:

132 - This schema is not valid for decoder-only models.

133 - """

134 -

135 - encoder_prompt: EncoderPrompt

136 - """The prompt for the encoder part of the model."""

137 -

138 - decoder_prompt: DecoderPrompt | None

139 - """

140 - The prompt for the decoder part of the model.

141 -

142 - Passing `None` will cause the prompt to be inferred automatically.

143 - """

144 -

145 -

146 -EncoderDecoderPrompt: TypeAlias = EncoderPrompt | ExplicitEncoderDecoderPrompt

147 -"""

148 -Schema for a prompt for an encoder-decoder model.

149 -

150 -You can pass a singleton encoder prompt, in which case the decoder prompt is

151 -considered to be `None` (i.e., infer automatically).

152 -"""

153 -

154 -

155 -SingletonPrompt: TypeAlias = DecoderOnlyPrompt | EncoderPrompt | DecoderPrompt

156 -"""

157 -Schema for a single prompt. This is as opposed to a data structure

158 -which encapsulates multiple prompts, such as

159 -[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt].

160 -"""

161 -

162 -

163 -PromptType: TypeAlias = DecoderOnlyPrompt | EncoderDecoderPrompt

164 -"""

165 -Schema for any prompt, regardless of model type.

166 -

167 -This is the input format accepted by most [`LLM`][vllm.entrypoints.llm.LLM] APIs.

168 -"""

169 -

170 -

171 -class DataPrompt(_PromptOptions):

172 - """

173 - Represents generic inputs that are converted to

174 - [`PromptType`][vllm.inputs.data.PromptType] by IO processor plugins.

175 - """

176 -

177 - data: Any

178 - """The input data."""

179 -

180 - data_format: str

181 - """The input data format."""

182 -

183 -

184 -# Outputs of processor

185 -class _InputOptions(TypedDict):

186 - """

187 - Additional options available to all input types.

188 - """

189 -

190 - arrival_time: NotRequired[float]

191 - """The time when the input was received (before rendering)."""

192 -

193 - cache_salt: NotRequired[str]

194 - """Optional cache salt to be used for prefix caching."""

195 -

196 -

197 -class TokenInputs(_InputOptions):

198 - """Represents token-based inputs."""

199 -

200 - type: Literal["token"]

201 - """The type of inputs."""

202 -

203 - prompt_token_ids: list[int]

204 - """The token IDs of the prompt."""

205 -

206 - prompt: NotRequired[str]

207 - """The prompt text corresponding to the token IDs, if available."""

208 -

209 -

210 -def token_inputs(

211 - prompt_token_ids: list[int],

212 - *,

213 - prompt: str | None = None,

214 - cache_salt: str | None = None,

215 -) -> TokenInputs:

216 - """Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional

217 - values."""

218 - inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)

219 -

220 - if prompt is not None:

221 - inputs["prompt"] = prompt

222 - if cache_salt is not None:

223 - inputs["cache_salt"] = cache_salt

224 -

225 - return inputs

226 -

227 -

228 -class EmbedsInputs(_InputOptions):

229 - """Represents embeddings-based inputs."""

230 -

231 - type: Literal["embeds"]

232 - """The type of inputs."""

233 -

234 - prompt_embeds: torch.Tensor

235 - """The embeddings of the prompt."""

236 -

237 - prompt: NotRequired[str]

238 - """The prompt text corresponding to the token IDs, if available."""

239 -

240 -

241 -def embeds_inputs(

242 - prompt_embeds: torch.Tensor,

243 - *,

244 - prompt: str | None = None,

245 - cache_salt: str | None = None,

246 -) -> EmbedsInputs:

247 - """Construct [`EmbedsInputs`][vllm.inputs.data.EmbedsInputs] from optional

248 - values."""

249 - inputs = EmbedsInputs(type="embeds", prompt_embeds=prompt_embeds)

250 -

251 - if prompt is not None:

252 - inputs["prompt"] = prompt

253 - if cache_salt is not None:

254 - inputs["cache_salt"] = cache_salt

255 -

256 - return inputs

257 -

258 -

259 -DecoderOnlyInputs: TypeAlias = TokenInputs | EmbedsInputs | MultiModalInputs

260 -"""

261 -A processed prompt from

262 -[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

263 -which can be passed to

264 -[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

265 -for decoder-only models.

266 -"""

267 -

268 -

269 -EncoderInputs: TypeAlias = TokenInputs | MultiModalEncDecInputs

270 -"""

271 -A processed encoder prompt from

272 -[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

273 -which can be passed to

274 -[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

275 -for encoder-decoder models.

276 -"""

277 -

278 -

279 -DecoderInputs: TypeAlias = TokenInputs | MultiModalInputs

280 -"""

281 -A processed decoder prompt from

282 -[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

283 -which can be passed to

284 -[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

285 -for encoder-decoder models.

286 -"""

287 -

288 -

289 -class EncoderDecoderInputs(TypedDict):

290 - """

291 - A processed pair of encoder and decoder singleton prompts.

292 - [`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

293 - which can be passed to

294 - [`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor]

295 - for encoder-decoder models.

296 - """

297 -

298 - type: Literal["enc_dec"]

299 -

300 - encoder_prompt: EncoderInputs

301 - """The inputs for the encoder portion."""

302 -

303 - decoder_prompt: DecoderInputs

304 - """The inputs for the decoder portion."""

305 -

306 - arrival_time: NotRequired[float]

307 - """The time when the input was received (before rendering)."""

308 -

309 -

310 -ProcessorInputs: TypeAlias = DecoderOnlyInputs | EncoderDecoderInputs

311 -"""

312 -A processed prompt from

313 -[`InputPreprocessor`][vllm.inputs.preprocess.InputPreprocessor]

314 -which can be passed to

315 -[`InputProcessor`][vllm.v1.engine.input_processor.InputProcessor].

316 -"""

317 -

318 -

319 -SingletonInputs: TypeAlias = DecoderOnlyInputs | MultiModalEncDecInputs

320 -"""The inputs for a single encoder/decoder prompt."""

321 -

322 -

323 -def _validate_enc_inputs(inputs: SingletonInputs) -> EncoderInputs:

324 - if inputs["type"] == "embeds":

325 - raise ValueError(

326 - "Embedding inputs are not supported for encoder-decoder models"

327 - )

328 -

329 - if inputs["type"] == "multimodal" and "encoder_prompt_token_ids" not in inputs:

330 - raise RuntimeError(

331 - "You should register an encoder-decoder multi-modal processor "

332 - "for encoder-decoder models."

333 - )

334 -

335 - return inputs # type: ignore[return-value]

336 -

337 -

338 -def _validate_dec_inputs(inputs: SingletonInputs) -> DecoderInputs:

339 - if inputs["type"] == "embeds":

340 - raise ValueError(

341 - "Embedding inputs are not supported for encoder-decoder models"

342 - )

343 -

344 - return inputs

345 -

346 -

347 -def _prepare_decoder_input_ids_for_generation(

348 - decoder_input_ids: list[int],

349 - decoder_start_token_id: int,

350 -) -> list[int]:

351 - """

352 - Prepare `decoder_input_ids` for generation with encoder-decoder models,

353 - according to `GenerationMixin._prepare_decoder_input_ids_for_generation()`.

354 -

355 - Source:

356 - https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/generation/utils.py

357 - """

358 - if len(decoder_input_ids) == 0 or decoder_input_ids[0] != decoder_start_token_id:

359 - decoder_input_ids = [decoder_start_token_id] + decoder_input_ids

360 -

361 - return decoder_input_ids

362 -

363 -

364 -def build_enc_dec_inputs(

365 - encoder_inputs: SingletonInputs,

366 - decoder_inputs: SingletonInputs | None,

367 - decoder_start_token_id: int,

368 - skip_decoder_start_token: bool = False,

369 -) -> EncoderDecoderInputs:

370 - enc_inputs = _validate_enc_inputs(encoder_inputs)

371 -

372 - if decoder_inputs is None:

373 - dec_inputs: DecoderInputs = enc_inputs

374 - else:

375 - dec_inputs = _validate_dec_inputs(decoder_inputs)

376 -

377 - enc_inputs_new: EncoderInputs

378 - dec_inputs_new: DecoderInputs

379 -

380 - if enc_inputs["type"] == "multimodal":

381 - from vllm.multimodal.inputs import mm_inputs

382 -

383 - enc_inputs_new = token_inputs(

384 - enc_inputs["encoder_prompt_token_ids"],

385 - prompt=enc_inputs.get("encoder_prompt"),

386 - )

387 - dec_inputs_new = mm_inputs(

388 - prompt_token_ids=dec_inputs["prompt_token_ids"],

389 - prompt=dec_inputs.get("prompt"),

390 - mm_kwargs=enc_inputs["mm_kwargs"],

391 - mm_hashes=enc_inputs["mm_hashes"],

392 - mm_placeholders=enc_inputs["mm_placeholders"],

393 - )

394 - elif enc_inputs["type"] == "token":

395 - enc_inputs_new = token_inputs(prompt_token_ids=[])

396 - dec_inputs_new = dec_inputs

397 - else:

398 - assert_never(enc_inputs)

399 -

400 - if not skip_decoder_start_token:

401 - dec_inputs_new["prompt_token_ids"] = _prepare_decoder_input_ids_for_generation(

402 - dec_inputs_new["prompt_token_ids"],

403 - decoder_start_token_id,

404 - )

405 -

406 - if cache_salt := enc_inputs.get("cache_salt"):

407 - dec_inputs_new["cache_salt"] = cache_salt

408 -

409 - return EncoderDecoderInputs(

410 - type="enc_dec",

411 - encoder_prompt=enc_inputs_new,

412 - decoder_prompt=dec_inputs_new,

413 - )

vllm/inputs/engine.py new

+352

@@ -0,0 +1,352 @@

1 +"""Schema and utilities for inputs to the engine client (`LLMEngine`/`AsyncLLM`)."""

2 +

3 +# SPDX-License-Identifier: Apache-2.0

4 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

5 +from collections.abc import Mapping, Sequence

6 +from typing import TYPE_CHECKING, Literal, TypeAlias

7 +

8 +from typing_extensions import NotRequired, TypedDict, assert_never

9 +

10 +if TYPE_CHECKING:

11 + import torch

12 +

13 + from vllm.multimodal.inputs import MultiModalKwargsOptionalItems, PlaceholderRange

14 +

15 +

16 +class _InputOptions(TypedDict):

17 + """

18 + Additional options available to all

19 + [`SingletonInput`][vllm.inputs.engine.SingletonInput] types.

20 + """

21 +

22 + arrival_time: NotRequired[float]

23 + """The time when the input was received (before rendering)."""

24 +

25 + cache_salt: NotRequired[str]

26 + """Optional cache salt to be used for prefix caching."""

27 +

28 +

29 +class TokensInput(_InputOptions):

30 + """Represents token-based input to the engine."""

31 +

32 + type: Literal["token"]

33 + """The type of input."""

34 +

35 + prompt_token_ids: list[int]

36 + """The token IDs of the prompt."""

37 +

38 + prompt: NotRequired[str]

39 + """The prompt text corresponding to the token IDs, if available."""

40 +

41 +

42 +def tokens_input(

43 + prompt_token_ids: list[int],

44 + *,

45 + prompt: str | None = None,

46 + cache_salt: str | None = None,

47 +) -> TokensInput:

48 + """

49 + Construct [`TokensInput`][vllm.inputs.engine.TokensInput]

50 + from optional values.

51 + """

52 + inputs = TokensInput(type="token", prompt_token_ids=prompt_token_ids)

53 +

54 + if prompt is not None:

55 + inputs["prompt"] = prompt

56 + if cache_salt is not None:

57 + inputs["cache_salt"] = cache_salt

58 +

59 + return inputs

60 +

61 +

62 +class EmbedsInput(_InputOptions):

63 + """Represents embeddings-based input to the engine."""

64 +

65 + type: Literal["embeds"]

66 + """The type of input."""

67 +

68 + prompt_embeds: "torch.Tensor"

69 + """The embeddings of the prompt."""

70 +

71 + prompt: NotRequired[str]

72 + """The prompt text corresponding to the token IDs, if available."""

73 +

74 +

75 +def embeds_input(

76 + prompt_embeds: "torch.Tensor",

77 + *,

78 + prompt: str | None = None,

79 + cache_salt: str | None = None,

80 +) -> EmbedsInput:

81 + """

82 + Construct [`EmbedsInput`][vllm.inputs.engine.EmbedsInput]

83 + from optional values.

84 + """

85 + inputs = EmbedsInput(type="embeds", prompt_embeds=prompt_embeds)

86 +

87 + if prompt is not None:

88 + inputs["prompt"] = prompt

89 + if cache_salt is not None:

90 + inputs["cache_salt"] = cache_salt

91 +

92 + return inputs

93 +

94 +

95 +MultiModalHashes: TypeAlias = Mapping[str, list[str]]

96 +"""

97 +A dictionary containing per-item hashes for each modality.

98 +"""

99 +

100 +

101 +MultiModalPlaceholders: TypeAlias = Mapping[str, Sequence["PlaceholderRange"]]

102 +"""

103 +A dictionary containing per-item placeholder ranges for each modality.

104 +"""

105 +

106 +

107 +class MultiModalInput(_InputOptions):

108 + """Represents multi-modal input to the engine."""

109 +

110 + type: Literal["multimodal"]

111 + """The type of input."""

112 +

113 + prompt_token_ids: list[int]

114 + """The processed token IDs which includes placeholder tokens."""

115 +

116 + prompt: NotRequired[str]

117 + """The prompt text corresponding to the token IDs, if available."""

118 +

119 + mm_kwargs: "MultiModalKwargsOptionalItems"

120 + """Keyword arguments to be directly passed to the model after batching."""

121 +

122 + mm_hashes: MultiModalHashes

123 + """The hashes of the multi-modal data."""

124 +

125 + mm_placeholders: MultiModalPlaceholders

126 + """

127 + For each modality, information about the placeholder tokens in

128 + `prompt_token_ids`.

129 + """

130 +

131 +

132 +def mm_input(

133 + prompt_token_ids: list[int],

134 + mm_kwargs: "MultiModalKwargsOptionalItems",

135 + mm_hashes: MultiModalHashes,

136 + mm_placeholders: MultiModalPlaceholders,

137 + *,

138 + prompt: str | None = None,

139 + cache_salt: str | None = None,

140 +) -> MultiModalInput:

141 + inputs = MultiModalInput(

142 + type="multimodal",

143 + prompt_token_ids=prompt_token_ids,

144 + mm_kwargs=mm_kwargs,

145 + mm_hashes=mm_hashes,

146 + mm_placeholders=mm_placeholders,

147 + )

148 +

149 + if prompt is not None:

150 + inputs["prompt"] = prompt

151 + if cache_salt is not None:

152 + inputs["cache_salt"] = cache_salt

153 +

154 + return inputs

155 +

156 +

157 +class MultiModalEncDecInput(MultiModalInput):

158 + """

159 + Represents multi-modal input to the engine for encoder-decoder models.

160 +

161 + Note:

162 + Even text-only encoder-decoder models are currently implemented

163 + as multi-modal models for convenience.

164 + (Example: https://github.com/vllm-project/bart-plugin)

165 + """

166 +

167 + encoder_prompt_token_ids: list[int]

168 + """The processed token IDs of the encoder prompt."""

169 +

170 + encoder_prompt: NotRequired[str]

171 + """The prompt text corresponding to the encoder token IDs, if available."""

172 +

173 +

174 +def mm_enc_dec_input(

175 + encoder_inputs: MultiModalInput,

176 + decoder_prompt_token_ids: list[int],

177 + *,

178 + decoder_prompt: str | None = None,

179 +) -> MultiModalEncDecInput:

180 + inputs = MultiModalEncDecInput(

181 + type="multimodal",

182 + prompt_token_ids=decoder_prompt_token_ids,

183 + encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],

184 + mm_kwargs=encoder_inputs["mm_kwargs"],

185 + mm_hashes=encoder_inputs["mm_hashes"],

186 + mm_placeholders=encoder_inputs["mm_placeholders"],

187 + )

188 +

189 + if decoder_prompt is not None:

190 + inputs["prompt"] = decoder_prompt

191 + if "prompt" in encoder_inputs:

192 + inputs["encoder_prompt"] = encoder_inputs["prompt"]

193 + if "cache_salt" in encoder_inputs:

194 + inputs["cache_salt"] = encoder_inputs["cache_salt"]

195 +

196 + return inputs

197 +

198 +

199 +DecoderOnlyEngineInput: TypeAlias = TokensInput | EmbedsInput | MultiModalInput

200 +"""

201 +A rendered [`DecoderOnlyPrompt`][vllm.inputs.llm.DecoderOnlyPrompt]

202 +which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.

203 +"""

204 +

205 +

206 +EncoderInput: TypeAlias = TokensInput | MultiModalEncDecInput

207 +"""

208 +A rendered [`EncoderPrompt`][vllm.inputs.llm.EncoderPrompt]

209 +which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.

210 +"""

211 +

212 +

213 +DecoderEngineInput: TypeAlias = TokensInput | MultiModalInput

214 +"""

215 +A rendered [`DecoderPrompt`][vllm.inputs.llm.DecoderPrompt]

216 +which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.

217 +"""

218 +

219 +

220 +class EncoderDecoderInput(TypedDict):

221 + """

222 + A rendered [`EncoderDecoderPrompt`][vllm.inputs.llm.EncoderDecoderPrompt]

223 + which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.

224 + """

225 +

226 + type: Literal["enc_dec"]

227 +

228 + encoder_prompt: EncoderInput

229 + """The inputs for the encoder portion."""

230 +

231 + decoder_prompt: DecoderEngineInput

232 + """The inputs for the decoder portion."""

233 +

234 + arrival_time: NotRequired[float]

235 + """The time when the input was received (before rendering)."""

236 +

237 +

238 +SingletonInput: TypeAlias = DecoderOnlyEngineInput | MultiModalEncDecInput

239 +"""

240 +A rendered [`SingletonPrompt`][vllm.inputs.llm.SingletonPrompt]

241 +which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.

242 +"""

243 +

244 +

245 +EngineInput: TypeAlias = DecoderOnlyEngineInput | EncoderDecoderInput

246 +"""

247 +A rendered [`PromptType`][vllm.inputs.llm.PromptType]

248 +which can be passed to `LLMEngine.add_request` or `AsyncLLM.add_request`.

249 +"""

250 +

251 +

252 +def _validate_enc_input(enc_input: SingletonInput) -> EncoderInput:

253 + if enc_input["type"] == "embeds":

254 + raise ValueError(

255 + "Embedding inputs are not supported for encoder-decoder models"

256 + )

257 +

258 + if (

259 + enc_input["type"] == "multimodal"

260 + and "encoder_prompt_token_ids" not in enc_input

261 + ):

262 + raise RuntimeError(

263 + "You should register an encoder-decoder multi-modal processor "

264 + "for encoder-decoder models."

265 + )

266 +

267 + return enc_input # type: ignore[return-value]

268 +

269 +

270 +def _validate_dec_input(dec_input: SingletonInput) -> DecoderEngineInput:

271 + if dec_input["type"] == "embeds":

272 + raise ValueError(

273 + "Embedding inputs are not supported for encoder-decoder models"

274 + )

275 +

276 + return dec_input

277 +

278 +

279 +def _prepare_decoder_input_ids_for_generation(

280 + decoder_input_ids: list[int],

281 + decoder_start_token_id: int,

282 +) -> list[int]:

283 + """

284 + Prepare `decoder_input_ids` for generation with encoder-decoder models,

285 + according to `GenerationMixin._prepare_decoder_input_ids_for_generation()`.

286 +

287 + Source:

288 + https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/generation/utils.py

289 + """

290 + if len(decoder_input_ids) == 0 or decoder_input_ids[0] != decoder_start_token_id:

291 + decoder_input_ids = [decoder_start_token_id] + decoder_input_ids

292 +

293 + return decoder_input_ids

294 +

295 +

296 +def build_enc_dec_input(

297 + encoder_input: SingletonInput,

298 + decoder_input: SingletonInput | None,

299 + decoder_start_token_id: int,

300 + skip_decoder_start_token: bool = False,

301 +) -> EncoderDecoderInput:

302 + enc_input = _validate_enc_input(encoder_input)

303 +

304 + if decoder_input is None:

305 + dec_input: DecoderEngineInput = enc_input

306 + else:

307 + dec_input = _validate_dec_input(decoder_input)

308 +

309 + enc_input_new: EncoderInput

310 + dec_input_new: DecoderEngineInput

311 +

312 + if enc_input["type"] == "multimodal":

313 + enc_input_new = tokens_input(

314 + enc_input["encoder_prompt_token_ids"],

315 + prompt=enc_input.get("encoder_prompt"),

316 + )

317 + dec_input_new = mm_input(

318 + prompt_token_ids=dec_input["prompt_token_ids"],

319 + prompt=dec_input.get("prompt"),

320 + mm_kwargs=enc_input["mm_kwargs"],

321 + mm_hashes=enc_input["mm_hashes"],

322 + mm_placeholders=enc_input["mm_placeholders"],

323 + )

324 + elif enc_input["type"] == "token":

325 + enc_input_new = tokens_input(prompt_token_ids=[])

326 + dec_input_new = dec_input

327 + else:

328 + assert_never(enc_input)

329 +

330 + if not skip_decoder_start_token:

331 + dec_input_new["prompt_token_ids"] = _prepare_decoder_input_ids_for_generation(

332 + dec_input_new["prompt_token_ids"],

333 + decoder_start_token_id,

334 + )

335 +

336 + if cache_salt := enc_input.get("cache_salt"):

337 + dec_input_new["cache_salt"] = cache_salt

338 +

339 + return EncoderDecoderInput(

340 + type="enc_dec",

341 + encoder_prompt=enc_input_new,

342 + decoder_prompt=dec_input_new,

343 + )

344 +

345 +

346 +def split_enc_dec_input(

347 + inputs: EngineInput,

348 +) -> tuple[SingletonInput | None, SingletonInput]:

349 + if inputs["type"] == "enc_dec":

350 + return inputs["encoder_prompt"], inputs["decoder_prompt"]

351 +

352 + return None, inputs

vllm/inputs/llm.py new

+222

@@ -0,0 +1,222 @@

1 +"""Schema and utilities for input prompts to the LLM API."""

2 +

3 +# SPDX-License-Identifier: Apache-2.0

4 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

5 +from collections.abc import Mapping, Sequence

6 +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, final

7 +

8 +from typing_extensions import NotRequired, TypedDict

9 +

10 +if TYPE_CHECKING:

11 + import torch

12 +

13 + from vllm.multimodal.inputs import AudioItem, ImageItem, VideoItem, VisionChunk

14 +

15 +

16 +_T = TypeVar("_T")

17 +

18 +ModalityData: TypeAlias = _T | list[_T | None] | None

19 +"""

20 +Either a single data item, or a list of data items. Can only be None if UUID

21 +is provided.

22 +

23 +The number of data items allowed per modality is restricted by

24 +`--limit-mm-per-prompt`.

25 +"""

26 +

27 +

28 +@final

29 +class MultiModalDataBuiltins(TypedDict, total=False):

30 + """Type annotations for modality types predefined by vLLM."""

31 +

32 + image: ModalityData["ImageItem"]

33 + """The input image(s)."""

34 +

35 + video: ModalityData["VideoItem"]

36 + """The input video(s)."""

37 +

38 + audio: ModalityData["AudioItem"]

39 + """The input audio(s)."""

40 +

41 + vision_chunk: ModalityData["VisionChunk"]

42 + """The input visual atom(s) - unified modality for images and video chunks."""

43 +

44 +

45 +MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]

46 +"""

47 +A dictionary containing an entry for each modality type to input.

48 +

49 +The built-in modalities are defined by

50 +[`MultiModalDataBuiltins`][vllm.inputs.llm.MultiModalDataBuiltins].

51 +"""

52 +

53 +MultiModalUUIDDict: TypeAlias = Mapping[str, Sequence[str | None] | str]

54 +"""

55 +A dictionary containing user-provided UUIDs for items in each modality.

56 +If a UUID for an item is not provided, its entry will be `None` and

57 +MultiModalHasher will compute a hash for the item.

58 +

59 +The UUID will be used to identify the item for all caching purposes

60 +(input processing caching, embedding caching, prefix caching, etc).

61 +"""

62 +

63 +

64 +class _PromptOptions(TypedDict):

65 + """

66 + Additional options available to all

67 + [`SingletonPrompt`][vllm.inputs.llm.SingletonPrompt] types.

68 + """

69 +

70 + multi_modal_data: NotRequired[MultiModalDataDict | None]

71 + """

72 + Optional multi-modal data to pass to the model,

73 + if the model supports it.

74 + """

75 +

76 + mm_processor_kwargs: NotRequired[dict[str, Any] | None]

77 + """

78 + Optional multi-modal processor kwargs to be forwarded to the

79 + multimodal input mapper & processor. Note that if multiple modalities

80 + have registered mappers etc for the model being considered, we attempt

81 + to pass the mm_processor_kwargs to each of them.

82 + """

83 +

84 + multi_modal_uuids: NotRequired[MultiModalUUIDDict]

85 + """

86 + Optional user-specified UUIDs for multimodal items, mapped by modality.

87 + Lists must match the number of items per modality and may contain `None`.

88 + For `None` entries, the hasher will compute IDs automatically; non-None

89 + entries override the default hashes for caching, and MUST be unique per

90 + multimodal item.

91 + """

92 +

93 + cache_salt: NotRequired[str]

94 + """

95 + Optional cache salt to be used for prefix caching.

96 + """

97 +

98 +

99 +class TextPrompt(_PromptOptions):

100 + """Schema for a text prompt."""

101 +

102 + prompt: str

103 + """The input text to be tokenized before passing to the model."""

104 +

105 +

106 +class TokensPrompt(_PromptOptions):

107 + """Schema for a tokenized prompt."""

108 +

109 + prompt_token_ids: list[int]

110 + """A list of token IDs to pass to the model."""

111 +

112 + prompt: NotRequired[str]

113 + """The prompt text corresponding to the token IDs, if available."""

114 +

115 + token_type_ids: NotRequired[list[int]]

116 + """A list of token type IDs to pass to the cross encoder model."""

117 +

118 +

119 +class EmbedsPrompt(_PromptOptions):

120 + """Schema for a prompt provided via token embeddings."""

121 +

122 + prompt_embeds: "torch.Tensor"

123 + """The embeddings of the prompt."""

124 +

125 + prompt: NotRequired[str]

126 + """The prompt text corresponding to the token embeddings, if available."""

127 +

128 +

129 +DecoderOnlyPrompt: TypeAlias = (

130 + str | TextPrompt | list[int] | TokensPrompt | EmbedsPrompt

131 +)

132 +"""

133 +Schema of a prompt for a decoder-only model:

134 +

135 +- A text prompt (string or [`TextPrompt`][vllm.inputs.llm.TextPrompt])

136 +- A tokenized prompt (list of token IDs, or

137 + [`TokensPrompt`][vllm.inputs.llm.TokensPrompt])

138 +- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.llm.EmbedsPrompt])

139 +

140 +For encoder-decoder models, passing a singleton prompt is shorthand for passing

141 +`ExplicitEncoderDecoderPrompt(encoder_prompt=prompt, decoder_prompt=None)`.

142 +"""

143 +

144 +

145 +EncoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt

146 +"""

147 +Schema of a prompt for the encoder part of a encoder-decoder model:

148 +

149 +- A text prompt (string or [`TextPrompt`][vllm.inputs.llm.TextPrompt])

150 +- A tokenized prompt (list of token IDs, or

151 + [`TokensPrompt`][vllm.inputs.llm.TokensPrompt])

152 +"""

153 +

154 +

155 +DecoderPrompt: TypeAlias = str | TextPrompt | list[int] | TokensPrompt

156 +"""

157 +Schema of a prompt for the decoder part of an encoder-decoder model:

158 +

159 +- A text prompt (string or [`TextPrompt`][vllm.inputs.llm.TextPrompt])

160 +- A tokenized prompt (list of token IDs, or

161 + [`TokensPrompt`][vllm.inputs.llm.TokensPrompt])

162 +

163 +Note:

164 + Multi-modal inputs are not supported for decoder prompts.

165 +"""

166 +

167 +

168 +class ExplicitEncoderDecoderPrompt(TypedDict):

169 + """

170 + Schema for a pair of encoder and decoder singleton prompts.

171 +

172 + Note:

173 + This schema is not valid for decoder-only models.

174 + """

175 +

176 + encoder_prompt: EncoderPrompt

177 + """The prompt for the encoder part of the model."""

178 +

179 + decoder_prompt: DecoderPrompt | None

180 + """

181 + The prompt for the decoder part of the model.

182 +

183 + Passing `None` will cause the prompt to be inferred automatically.

184 + """

185 +

186 +

187 +EncoderDecoderPrompt: TypeAlias = EncoderPrompt | ExplicitEncoderDecoderPrompt

188 +"""

189 +Schema for a prompt for an encoder-decoder model.

190 +

191 +You can pass a singleton encoder prompt, in which case the decoder prompt is

192 +considered to be `None` (i.e., infer automatically).

193 +"""

194 +

195 +

196 +SingletonPrompt: TypeAlias = DecoderOnlyPrompt | EncoderPrompt | DecoderPrompt

197 +"""

198 +Schema for a single prompt. This is as opposed to a data structure

199 +which encapsulates multiple prompts, such as

200 +[`ExplicitEncoderDecoderPrompt`][vllm.inputs.llm.ExplicitEncoderDecoderPrompt].

201 +"""

202 +

203 +

204 +PromptType: TypeAlias = DecoderOnlyPrompt | EncoderDecoderPrompt

205 +"""

206 +Schema for any prompt, regardless of model type.

207 +

208 +This is the input format accepted by most [`LLM`][vllm.entrypoints.llm.LLM] APIs.

209 +"""

210 +

211 +

212 +class DataPrompt(_PromptOptions):

213 + """

214 + Represents generic inputs that are converted to

215 + [`PromptType`][vllm.inputs.llm.PromptType] by IO processor plugins.

216 + """

217 +

218 + data: Any

219 + """The input data."""

220 +

221 + data_format: str

222 + """The input data format."""

vllm/inputs/parse.py deleted

-13

@@ -1,13 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -from .data import ProcessorInputs, SingletonInputs

5 -

6 -

7 -def split_enc_dec_inputs(

8 - inputs: ProcessorInputs,

9 -) -> tuple[SingletonInputs | None, SingletonInputs]:

10 - if inputs["type"] == "enc_dec":

11 - return inputs["encoder_prompt"], inputs["decoder_prompt"]

12 -

13 - return None, inputs

vllm/inputs/preprocess.py

+34 -73

@@ -7,14 +7,9 @@ from typing import Any, overload

7 from typing_extensions import assert_never

8

9 from vllm.config import VllmConfig

10 -from vllm.inputs.data import build_enc_dec_inputs

10 +from vllm.inputs import build_enc_dec_input

11 from vllm.logger import init_logger

12 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

13 -from vllm.multimodal.inputs import (

14 - MultiModalDataDict,

15 - MultiModalInputs,

16 - MultiModalUUIDDict,

17 -)

13 from vllm.renderers import BaseRenderer, renderer_from_config

14 from vllm.renderers.inputs import (

15 DecoderDictPrompt,

@@ -26,20 +21,25 @@ from vllm.renderers.inputs import (

21 from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt

22 from vllm.tokenizers import TokenizerLike

23

29 -from .data import (

30 - DecoderInputs,

31 - DecoderOnlyInputs,

32 - EmbedsInputs,

24 +from .engine import (

25 + DecoderEngineInput,

26 + DecoderOnlyEngineInput,

27 + EmbedsInput,

28 + EncoderDecoderInput,

29 + EncoderInput,

30 + EngineInput,

31 + MultiModalInput,

32 + SingletonInput,

33 + TokensInput,

34 + tokens_input,

35 +)

36 +from .llm import (

37 EmbedsPrompt,

34 - EncoderDecoderInputs,

35 - EncoderInputs,

36 - ProcessorInputs,

38 + MultiModalDataDict,

39 + MultiModalUUIDDict,

40 PromptType,

38 - SingletonInputs,

41 TextPrompt,

40 - TokenInputs,

42 TokensPrompt,

42 - token_inputs,

43 )

44

45 logger = init_logger(__name__)

@@ -95,7 +95,7 @@ class InputPreprocessor:

95 tokenization_kwargs: dict[str, Any] | None = None,

96 *,

97 mm_uuids: MultiModalUUIDDict | None = None,

98 - ) -> MultiModalInputs:

98 + ) -> MultiModalInput:

99 """

100 Apply the model's multi-modal processor to a multi-modal prompt,

101 returning the corresponding token IDs and metadata.

@@ -111,7 +111,7 @@ class InputPreprocessor:

111 def _process_embeds(

112 self,

113 parsed_content: EmbedsPrompt,

114 - ) -> EmbedsInputs:

114 + ) -> EmbedsInput:

115 return self.renderer._process_embeds(parsed_content)

116

117 def _truncate_inputs(

@@ -134,12 +134,12 @@ class InputPreprocessor:

134 self,

135 parsed_content: TokensPrompt,

136 tokenization_kwargs: dict[str, Any] | None = None,

137 - ) -> TokenInputs | MultiModalInputs:

137 + ) -> TokensInput | MultiModalInput:

138 prompt_token_ids = self._truncate_inputs(

139 parsed_content["prompt_token_ids"], tokenization_kwargs

140 )

141

142 - inputs: TokenInputs | MultiModalInputs

142 + inputs: TokensInput | MultiModalInput

143 if multi_modal_data := parsed_content.get("multi_modal_data"):

144 inputs = self._process_multimodal(

145 prompt_token_ids,

@@ -149,7 +149,7 @@ class InputPreprocessor:

149 mm_uuids=parsed_content.get("multi_modal_uuids"),

150 )

151 else:

152 - inputs = token_inputs(prompt_token_ids)

152 + inputs = tokens_input(prompt_token_ids)

153

154 if prompt_text := parsed_content.get("prompt"):

155 inputs["prompt"] = prompt_text

@@ -162,10 +162,10 @@ class InputPreprocessor:

162 self,

163 parsed_content: TextPrompt,

164 tokenization_kwargs: dict[str, Any] | None = None,

165 - ) -> TokenInputs | MultiModalInputs:

165 + ) -> TokensInput | MultiModalInput:

166 prompt_text = parsed_content["prompt"]

167

168 - inputs: TokenInputs | MultiModalInputs

168 + inputs: TokensInput | MultiModalInput

169 if multi_modal_data := parsed_content.get("multi_modal_data"):

170 inputs = self._process_multimodal(

171 prompt_text,

@@ -178,7 +178,7 @@ class InputPreprocessor:

178 prompt_text,

179 tokenization_kwargs=tokenization_kwargs,

180 )

181 - inputs = token_inputs(prompt_token_ids)

181 + inputs = tokens_input(prompt_token_ids)

182

183 inputs["prompt"] = prompt_text

184

@@ -192,38 +192,27 @@ class InputPreprocessor:

192 self,

193 prompt: EncoderDictPrompt,

194 tokenization_kwargs: dict[str, Any] | None = None,

195 - ) -> EncoderInputs: ...

195 + ) -> EncoderInput: ...

196

197 @overload

198 def _prompt_to_llm_inputs( # type: ignore[misc]

199 self,

200 prompt: DecoderDictPrompt,

201 tokenization_kwargs: dict[str, Any] | None = None,

202 - ) -> DecoderInputs: ...

202 + ) -> DecoderEngineInput: ...

203

204 @overload

205 def _prompt_to_llm_inputs( # type: ignore[misc]

206 self,

207 prompt: DecoderOnlyDictPrompt,

208 tokenization_kwargs: dict[str, Any] | None = None,

209 - ) -> DecoderOnlyInputs: ...

209 + ) -> DecoderOnlyEngineInput: ...

210

211 def _prompt_to_llm_inputs(

212 self,

213 prompt: SingletonDictPrompt,

214 tokenization_kwargs: dict[str, Any] | None = None,

215 - ) -> SingletonInputs:

216 - """

217 - Extract the singleton inputs from a prompt.

218 -

219 - Arguments:

220 -

221 - * prompt: single encoder or decoder input prompt

222 -

223 - Returns:

224 -

225 - * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance

226 - """

215 + ) -> SingletonInput:

216 if "prompt_embeds" in prompt:

217 return self._process_embeds(prompt) # type: ignore[arg-type]

218

@@ -242,22 +231,7 @@ class InputPreprocessor:

231 self,

232 prompt: EncoderDecoderDictPrompt,

233 tokenization_kwargs: dict[str, Any] | None = None,

245 - ) -> EncoderDecoderInputs:

246 - """

247 - For encoder/decoder models only:

248 - Process an input prompt into an

249 - [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]

250 - instance.

251 -

252 - Arguments:

253 -

254 - * prompt: an input prompt

255 -

256 - Returns:

257 -

258 - * [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]

259 - instance

260 - """

234 + ) -> EncoderDecoderInput:

235 encoder_prompt = prompt["encoder_prompt"]

236 decoder_prompt = prompt["decoder_prompt"]

237

@@ -270,12 +244,12 @@ class InputPreprocessor:

244 self.renderer.mm_processor.skip_decoder_start_token

245 )

246

273 - return build_enc_dec_inputs(

274 - encoder_inputs=self._prompt_to_llm_inputs(

247 + return build_enc_dec_input(

248 + encoder_input=self._prompt_to_llm_inputs(

249 encoder_prompt,

250 tokenization_kwargs=tokenization_kwargs,

251 ),

278 - decoder_inputs=(

252 + decoder_input=(

253 None

254 if decoder_prompt is None

255 else self._prompt_to_llm_inputs(

@@ -291,20 +265,7 @@ class InputPreprocessor:

265 self,

266 prompt: DecoderOnlyDictPrompt,

267 tokenization_kwargs: dict[str, Any] | None = None,

294 - ) -> DecoderOnlyInputs:

295 - """

296 - For decoder-only models:

297 - Process an input prompt into a

298 - [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance.

299 -

300 - Arguments:

301 -

302 - * prompt: input prompt

303 -

304 - Returns:

305 -

306 - * [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance

307 - """

268 + ) -> DecoderOnlyEngineInput:

269 return self._prompt_to_llm_inputs(

270 prompt,

271 tokenization_kwargs=tokenization_kwargs,

@@ -314,7 +275,7 @@ class InputPreprocessor:

275 self,

276 prompt: PromptType,

277 tokenization_kwargs: dict[str, Any] | None = None,

317 - ) -> ProcessorInputs:

278 + ) -> EngineInput:

279 """Preprocess the input prompt."""

280 if self.model_config.is_encoder_decoder:

281 # Encoder-decoder model requires special mapping of

vllm/model_executor/models/aria.py

+1 -1

@@ -12,6 +12,7 @@ from transformers.models.aria.processing_aria import AriaProcessor

12 from vllm.config import VllmConfig

13 from vllm.config.multimodal import BaseDummyOptions

14 from vllm.distributed import get_tensor_model_parallel_rank

15 +from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.layers.activation import get_act_fn

17 from vllm.model_executor.layers.fused_moe import SharedFusedMoE

18 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -24,7 +25,6 @@ from vllm.model_executor.model_loader.weight_utils import (

25 )

26 from vllm.multimodal import MULTIMODAL_REGISTRY

27 from vllm.multimodal.inputs import (

27 - MultiModalDataDict,

28 MultiModalFieldConfig,

29 MultiModalKwargsItems,

30 )

vllm/model_executor/models/audioflamingo3.py

+1 -2

@@ -31,17 +31,16 @@ from transformers.models.qwen2_audio import Qwen2AudioEncoder

31

32 from vllm.config import VllmConfig

33 from vllm.config.multimodal import BaseDummyOptions

34 +from vllm.inputs import ModalityData, MultiModalDataDict

35 from vllm.model_executor.layers.activation import get_act_fn

36 from vllm.model_executor.models.module_mapping import MultiModelKeys

37 from vllm.multimodal import MULTIMODAL_REGISTRY

38 from vllm.multimodal.inputs import (

38 - MultiModalDataDict,

39 MultiModalFieldConfig,

40 MultiModalKwargsItems,

41 )

42 from vllm.multimodal.parse import (

43 DictEmbeddingItems,

44 - ModalityData,

44 ModalityDataItems,

45 MultiModalDataItems,

46 MultiModalDataParser,

vllm/model_executor/models/aya_vision.py

+1 -1

@@ -17,9 +17,9 @@ from transformers.models.got_ocr2.image_processing_got_ocr2 import (

17

18 from vllm.config import VllmConfig

19 from vllm.config.multimodal import BaseDummyOptions

20 +from vllm.inputs import MultiModalDataDict

21 from vllm.multimodal import MULTIMODAL_REGISTRY

22 from vllm.multimodal.inputs import (

22 - MultiModalDataDict,

23 MultiModalFieldConfig,

24 MultiModalKwargsItems,

25 )

vllm/model_executor/models/bagel.py

+1 -1

@@ -15,6 +15,7 @@ import torch.nn as nn

15

16 from vllm.config import VllmConfig

17 from vllm.config.multimodal import BaseDummyOptions

18 +from vllm.inputs import MultiModalDataDict

19 from vllm.logger import init_logger

20 from vllm.model_executor.layers.activation import get_act_fn

21 from vllm.model_executor.layers.linear import (

@@ -24,7 +25,6 @@ from vllm.model_executor.layers.linear import (

25 from vllm.model_executor.layers.quantization import QuantizationConfig

26 from vllm.multimodal import MULTIMODAL_REGISTRY

27 from vllm.multimodal.inputs import (

27 - MultiModalDataDict,

28 MultiModalFieldConfig,

29 MultiModalKwargsItems,

30 )

vllm/model_executor/models/bee.py

+1 -1

@@ -9,8 +9,8 @@ from transformers.activations import GELUActivation

9

10 from vllm.config import VllmConfig

11 from vllm.config.multimodal import BaseDummyOptions

12 +from vllm.inputs import MultiModalDataDict

13 from vllm.multimodal import MULTIMODAL_REGISTRY

13 -from vllm.multimodal.inputs import MultiModalDataDict

14

15 from .llava_next import (

16 LlavaDummyInputsBuilder,

vllm/model_executor/models/blip2.py

+1 -1

@@ -15,11 +15,11 @@ from transformers import (

15

16 from vllm.config import CacheConfig, VllmConfig

17 from vllm.config.multimodal import BaseDummyOptions

18 +from vllm.inputs import MultiModalDataDict

19 from vllm.model_executor.layers.activation import get_act_fn

20 from vllm.model_executor.layers.quantization import QuantizationConfig

21 from vllm.multimodal import MULTIMODAL_REGISTRY

22 from vllm.multimodal.inputs import (

22 - MultiModalDataDict,

23 MultiModalFieldConfig,

24 MultiModalKwargsItems,

25 )

vllm/model_executor/models/chameleon.py

+1 -1

@@ -19,6 +19,7 @@ from transformers import (

19 from vllm.config import CacheConfig, VllmConfig

20 from vllm.config.multimodal import BaseDummyOptions

21 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size

22 +from vllm.inputs import MultiModalDataDict

23 from vllm.logger import init_logger

24 from vllm.model_executor.layers.activation import SiluAndMul

25 from vllm.model_executor.layers.attention import Attention

@@ -43,7 +44,6 @@ from vllm.model_executor.model_loader.weight_utils import (

44 from vllm.model_executor.utils import set_weight_attrs

45 from vllm.multimodal import MULTIMODAL_REGISTRY

46 from vllm.multimodal.inputs import (

46 - MultiModalDataDict,

47 MultiModalFieldConfig,

48 MultiModalKwargsItems,

49 )

vllm/model_executor/models/clip.py

+2 -3

@@ -17,6 +17,7 @@ from transformers import (

17 from vllm.config import VllmConfig

18 from vllm.config.multimodal import BaseDummyOptions

19 from vllm.distributed import divide, get_tensor_model_parallel_world_size

20 +from vllm.inputs import MultiModalDataDict, MultiModalInput

21 from vllm.model_executor.layers.activation import get_act_fn

22 from vllm.model_executor.layers.attention import Attention, MMEncoderAttention

23 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -32,9 +33,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

33 from vllm.model_executor.models.interfaces import SupportsQuant

34 from vllm.multimodal import MULTIMODAL_REGISTRY

35 from vllm.multimodal.inputs import (

35 - MultiModalDataDict,

36 MultiModalFieldConfig,

37 - MultiModalInputs,

37 MultiModalKwargsItems,

38 )

39 from vllm.multimodal.parse import (

@@ -207,7 +206,7 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):

206 self,

207 inputs: ProcessorInputs,

208 timing_ctx: TimingContext,

210 - ) -> MultiModalInputs:

209 + ) -> MultiModalInput:

210 if inputs.mm_data_items:

211 if isinstance(inputs.prompt, str):

212 if len(inputs.prompt) > 0:

vllm/model_executor/models/cohere2_vision.py

+1 -1

@@ -19,6 +19,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import (

19

20 from vllm.config import VllmConfig

21 from vllm.config.multimodal import BaseDummyOptions

22 +from vllm.inputs import MultiModalDataDict

23 from vllm.model_executor.layers.activation import MulAndSilu

24 from vllm.model_executor.layers.linear import (

25 MergedColumnParallelLinear,

@@ -28,7 +29,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

29 from vllm.model_executor.layers.quantization.awq import AWQConfig

30 from vllm.multimodal import MULTIMODAL_REGISTRY

31 from vllm.multimodal.inputs import (

31 - MultiModalDataDict,

32 MultiModalFieldConfig,

33 MultiModalKwargsItems,

34 )

vllm/model_executor/models/cohere_asr.py

+6 -10

@@ -3,7 +3,7 @@

3

4 import math

5 from collections.abc import Iterable, Mapping, Sequence

6 -from typing import Literal, cast

6 +from typing import Literal

7

8 import numpy as np

9 import torch

@@ -14,7 +14,7 @@ from transformers import PretrainedConfig

14 from vllm.compilation.decorators import support_torch_compile

15 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

16 from vllm.distributed import get_tensor_model_parallel_world_size

17 -from vllm.inputs.data import PromptType

17 +from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt

18 from vllm.logger import init_logger

19 from vllm.model_executor.layers.activation import get_act_fn

20 from vllm.model_executor.layers.attention import (

@@ -32,7 +32,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

32 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

33 from vllm.multimodal import MULTIMODAL_REGISTRY

34 from vllm.multimodal.inputs import (

35 - MultiModalDataDict,

35 MultiModalFieldConfig,

36 MultiModalKwargsItems,

37 )

@@ -2047,14 +2046,11 @@ class CohereASRForConditionalGeneration(

2046 f"<|noitn|><|notimestamp|><|nodiarize|>"

2047 )

2048 prompt_text = request_prompt if request_prompt else default_prompt

2050 - prompt = {

2051 - "prompt": prompt_text,

2052 - "multi_modal_data": {

2053 - "audio": (audio, stt_config.sample_rate),

2054 - },

2055 - }

2049

2057 - return cast(PromptType, prompt)

2050 + return TextPrompt(

2051 + prompt=prompt_text,

2052 + multi_modal_data={"audio": (audio, stt_config.sample_rate)},

2053 + )

2054

2055 @classmethod

2056 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

vllm/model_executor/models/colmodernvbert.py

+1 -1

@@ -16,11 +16,11 @@ from transformers import BatchFeature

16

17 from vllm.config import VllmConfig

18 from vllm.config.multimodal import BaseDummyOptions

19 +from vllm.inputs import MultiModalDataDict

20 from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed

21 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

22 from vllm.multimodal import MULTIMODAL_REGISTRY

23 from vllm.multimodal.inputs import (

23 - MultiModalDataDict,

24 MultiModalFieldConfig,

25 MultiModalKwargsItems,

26 )

vllm/model_executor/models/deepseek_ocr.py

+1 -1

@@ -12,6 +12,7 @@ from transformers import BatchFeature, CLIPVisionConfig

12

13 from vllm.config import VllmConfig

14 from vllm.config.multimodal import BaseDummyOptions

15 +from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.models.interfaces import (

17 MultiModalEmbeddings,

18 SupportsLoRA,

@@ -27,7 +28,6 @@ from vllm.model_executor.models.utils import (

28 )

29 from vllm.multimodal import MULTIMODAL_REGISTRY

30 from vllm.multimodal.inputs import (

30 - MultiModalDataDict,

31 MultiModalFieldConfig,

32 MultiModalKwargsItems,

33 NestedTensors,

vllm/model_executor/models/deepseek_ocr2.py

+1 -1

@@ -12,6 +12,7 @@ from transformers import BatchFeature

12

13 from vllm.config import VllmConfig

14 from vllm.config.multimodal import BaseDummyOptions

15 +from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.models.interfaces import (

17 MultiModalEmbeddings,

18 SupportsLoRA,

@@ -27,7 +28,6 @@ from vllm.model_executor.models.utils import (

28 )

29 from vllm.multimodal import MULTIMODAL_REGISTRY

30 from vllm.multimodal.inputs import (

30 - MultiModalDataDict,

31 MultiModalFieldConfig,

32 MultiModalKwargsItems,

33 NestedTensors,

vllm/model_executor/models/deepseek_vl2.py

+1 -1

@@ -17,11 +17,11 @@ from transformers import BatchFeature

17 from vllm.config import VllmConfig

18 from vllm.config.multimodal import BaseDummyOptions

19 from vllm.distributed import get_tensor_model_parallel_world_size

20 +from vllm.inputs import MultiModalDataDict

21 from vllm.model_executor.layers.quantization import QuantizationConfig

22 from vllm.model_executor.models.transformers.utils import replace_linear_class

23 from vllm.multimodal import MULTIMODAL_REGISTRY

24 from vllm.multimodal.inputs import (

24 - MultiModalDataDict,

25 MultiModalFieldConfig,

26 MultiModalKwargsItems,

27 )

vllm/model_executor/models/dots_ocr.py

+1 -1

@@ -15,6 +15,7 @@ from vllm.distributed.parallel_state import (

15 get_tensor_model_parallel_rank,

16 get_tensor_model_parallel_world_size,

17 )

18 +from vllm.inputs import MultiModalDataDict

19 from vllm.model_executor.layers.activation import SiluAndMul

20 from vllm.model_executor.layers.attention import (

21 MMEncoderAttention,

@@ -54,7 +55,6 @@ from vllm.model_executor.models.utils import (

55 )

56 from vllm.model_executor.models.vision import get_vit_attn_backend

57 from vllm.multimodal import MULTIMODAL_REGISTRY

57 -from vllm.multimodal.inputs import MultiModalDataDict

58 from vllm.sequence import IntermediateTensors

59 from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionConfig

60 from vllm.utils.tensor_schema import TensorSchema, TensorShape

vllm/model_executor/models/ernie45_vl.py

+1 -1

@@ -40,6 +40,7 @@ from vllm.config import VllmConfig

40 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

41 from vllm.distributed import parallel_state

42 from vllm.distributed import utils as dist_utils

43 +from vllm.inputs import MultiModalDataDict

44 from vllm.logger import init_logger

45 from vllm.model_executor.layers.activation import QuickGELU

46 from vllm.model_executor.layers.attention import (

@@ -58,7 +59,6 @@ from vllm.model_executor.layers.rotary_embedding.common import (

59 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

60 from vllm.multimodal import MULTIMODAL_REGISTRY

61 from vllm.multimodal.inputs import (

61 - MultiModalDataDict,

62 MultiModalFeatureSpec,

63 MultiModalFieldConfig,

64 MultiModalKwargsItems,

vllm/model_executor/models/fireredasr2.py

+1 -2

@@ -15,7 +15,7 @@ from transformers import (

15

16 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

17 from vllm.config.multimodal import BaseDummyOptions

18 -from vllm.inputs.data import PromptType

18 +from vllm.inputs import MultiModalDataDict, PromptType

19 from vllm.logger import init_logger

20 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

21 from vllm.model_executor.layers.linear import (

@@ -27,7 +27,6 @@ from vllm.model_executor.models.whisper_utils import (

27 )

28 from vllm.multimodal import MULTIMODAL_REGISTRY

29 from vllm.multimodal.inputs import (

30 - MultiModalDataDict,

30 MultiModalFieldConfig,

31 MultiModalKwargsItems,

32 )

vllm/model_executor/models/funasr.py

+1 -2

@@ -17,7 +17,7 @@ from transformers import (

17 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

18 from vllm.config.multimodal import BaseDummyOptions

19 from vllm.distributed import get_tensor_model_parallel_world_size

20 -from vllm.inputs.data import PromptType

20 +from vllm.inputs import MultiModalDataDict, PromptType

21 from vllm.logger import init_logger

22 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

23 from vllm.model_executor.layers.attention.mm_encoder_attention import (

@@ -37,7 +37,6 @@ from vllm.model_executor.models.whisper_utils import (

37 )

38 from vllm.multimodal import MULTIMODAL_REGISTRY

39 from vllm.multimodal.inputs import (

40 - MultiModalDataDict,

40 MultiModalFieldConfig,

41 MultiModalKwargsItems,

42 )

vllm/model_executor/models/funaudiochat.py

+1 -1

@@ -27,12 +27,12 @@ from transformers.modeling_outputs import BaseModelOutput

27

28 from vllm.config import VllmConfig

29 from vllm.config.multimodal import BaseDummyOptions

30 +from vllm.inputs import MultiModalDataDict

31 from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

32 from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear

33 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

34 from vllm.multimodal import MULTIMODAL_REGISTRY

35 from vllm.multimodal.inputs import (

35 - MultiModalDataDict,

36 MultiModalFieldConfig,

37 MultiModalKwargsItems,

38 )

vllm/model_executor/models/fuyu.py

+1 -1

@@ -28,11 +28,11 @@ from transformers import BatchFeature, FuyuConfig, FuyuImageProcessor, FuyuProce

28

29 from vllm.config import VllmConfig

30 from vllm.config.multimodal import BaseDummyOptions

31 +from vllm.inputs import MultiModalDataDict

32 from vllm.model_executor.layers.linear import ColumnParallelLinear

33 from vllm.model_executor.models.persimmon import PersimmonForCausalLM

34 from vllm.multimodal import MULTIMODAL_REGISTRY

35 from vllm.multimodal.inputs import (

35 - MultiModalDataDict,

36 MultiModalFieldConfig,

37 MultiModalKwargsItems,

38 )

vllm/model_executor/models/gemma3_mm.py

+1 -1

@@ -12,12 +12,12 @@ from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs

12

13 from vllm.config import VllmConfig

14 from vllm.config.multimodal import BaseDummyOptions

15 +from vllm.inputs import MultiModalDataDict

16 from vllm.logger import init_logger

17 from vllm.model_executor.layers.layernorm import GemmaRMSNorm

18 from vllm.model_executor.models.module_mapping import MultiModelKeys

19 from vllm.multimodal import MULTIMODAL_REGISTRY

20 from vllm.multimodal.inputs import (

20 - MultiModalDataDict,

21 MultiModalFieldConfig,

22 MultiModalKwargsItems,

23 )

vllm/model_executor/models/gemma3n_mm.py

+1 -2

@@ -19,7 +19,7 @@ from transformers.models.siglip import SiglipImageProcessorFast

19

20 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

21 from vllm.config.multimodal import BaseDummyOptions

22 -from vllm.inputs.data import PromptType, TextPrompt

22 +from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt

23 from vllm.logger import init_logger

24 from vllm.model_executor.layers.layernorm import RMSNorm

25 from vllm.model_executor.layers.linear import RowParallelLinear

@@ -32,7 +32,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

32 from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS

33 from vllm.multimodal import MULTIMODAL_REGISTRY

34 from vllm.multimodal.inputs import (

35 - MultiModalDataDict,

35 MultiModalFieldConfig,

36 MultiModalKwargsItems,

37 )

vllm/model_executor/models/glm4_1v.py

+1 -1

@@ -50,6 +50,7 @@ from vllm.config import VllmConfig

50 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

51 from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state

52 from vllm.distributed import utils as dist_utils

53 +from vllm.inputs import MultiModalDataDict

54 from vllm.logger import init_logger

55 from vllm.model_executor.layers.attention import (

56 MMEncoderAttention,

@@ -74,7 +75,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

75 from vllm.model_executor.models.module_mapping import MultiModelKeys

76 from vllm.multimodal import MULTIMODAL_REGISTRY

77 from vllm.multimodal.inputs import (

77 - MultiModalDataDict,

78 MultiModalFeatureSpec,

79 MultiModalFieldConfig,

80 MultiModalKwargsItems,

vllm/model_executor/models/glm4v.py

+1 -1

@@ -18,6 +18,7 @@ from transformers import BatchFeature

18 from vllm.config import VllmConfig

19 from vllm.config.multimodal import BaseDummyOptions

20 from vllm.distributed import get_tensor_model_parallel_world_size

21 +from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn

23 from vllm.model_executor.layers.attention import MMEncoderAttention

24 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -32,7 +33,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

33 from vllm.model_executor.models.module_mapping import MultiModelKeys

34 from vllm.multimodal import MULTIMODAL_REGISTRY

35 from vllm.multimodal.inputs import (

35 - MultiModalDataDict,

36 MultiModalFeatureSpec,

37 MultiModalFieldConfig,

38 MultiModalKwargsItems,

vllm/model_executor/models/glmasr.py

+1 -3

@@ -14,7 +14,7 @@ from transformers.models.whisper import WhisperFeatureExtractor

14 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

15 from vllm.config.multimodal import BaseDummyOptions

16 from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size

17 -from vllm.inputs.data import PromptType, TokensPrompt

17 +from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt

18 from vllm.model_executor.layers.activation import get_act_fn

19 from vllm.model_executor.layers.attention import MMEncoderAttention

20 from vllm.model_executor.layers.linear import (

@@ -27,13 +27,11 @@ from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb

27 from vllm.model_executor.models.module_mapping import MultiModelKeys

28 from vllm.multimodal import MULTIMODAL_REGISTRY

29 from vllm.multimodal.inputs import (

30 - MultiModalDataDict,

30 MultiModalFieldConfig,

31 MultiModalKwargsItems,

32 )

33 from vllm.multimodal.parse import (

34 DictEmbeddingItems,

36 - ModalityData,

35 ModalityDataItems,

36 MultiModalDataItems,

37 MultiModalDataParser,

vllm/model_executor/models/granite_speech.py

+1 -2

@@ -36,13 +36,12 @@ from transformers import BatchFeature, PretrainedConfig

36

37 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

38 from vllm.config.multimodal import BaseDummyOptions

39 -from vllm.inputs.data import PromptType, TokensPrompt

39 +from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt

40 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

41 from vllm.model_executor.layers.quantization import QuantizationConfig

42 from vllm.model_executor.models.module_mapping import MultiModelKeys

43 from vllm.multimodal import MULTIMODAL_REGISTRY

44 from vllm.multimodal.inputs import (

45 - MultiModalDataDict,

45 MultiModalFieldConfig,

46 MultiModalKwargsItems,

47 )

vllm/model_executor/models/hunyuan_vision.py

+1 -2

@@ -37,6 +37,7 @@ from vllm.config import VllmConfig

37 from vllm.config.multimodal import BaseDummyOptions

38 from vllm.distributed import parallel_state

39 from vllm.distributed import utils as dist_utils

40 +from vllm.inputs import ModalityData, MultiModalDataDict

41 from vllm.logger import init_logger

42 from vllm.model_executor.layers.activation import get_act_fn

43 from vllm.model_executor.layers.attention import MMEncoderAttention

@@ -52,8 +53,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

53 from vllm.multimodal import MULTIMODAL_REGISTRY

54 from vllm.multimodal.inputs import (

55 ImageItem,

55 - ModalityData,

56 - MultiModalDataDict,

56 MultiModalFeatureSpec,

57 MultiModalFieldConfig,

58 MultiModalKwargsItems,

vllm/model_executor/models/hyperclovax_vision.py

+1 -1

@@ -18,10 +18,10 @@ from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig

18

19 from vllm.config import VllmConfig

20 from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.quantization import QuantizationConfig

23 from vllm.multimodal import MULTIMODAL_REGISTRY

24 from vllm.multimodal.inputs import (

24 - MultiModalDataDict,

25 MultiModalFieldConfig,

26 MultiModalKwargsItems,

27 )

vllm/model_executor/models/hyperclovax_vision_v2.py

+1 -1

@@ -21,9 +21,9 @@ from transformers import BatchFeature

21 from vllm.config import VllmConfig

22 from vllm.config.multimodal import BaseDummyOptions

23 from vllm.forward_context import set_forward_context

24 +from vllm.inputs import MultiModalDataDict

25 from vllm.multimodal import MULTIMODAL_REGISTRY

26 from vllm.multimodal.inputs import (

26 - MultiModalDataDict,

27 MultiModalFieldConfig,

28 MultiModalKwargsItems,

29 )

vllm/model_executor/models/idefics3.py

+1 -1

@@ -30,6 +30,7 @@ from transformers import (

30

31 from vllm.config import VllmConfig

32 from vllm.config.multimodal import BaseDummyOptions

33 +from vllm.inputs import MultiModalDataDict

34 from vllm.model_executor.layers.linear import ReplicatedLinear

35 from vllm.model_executor.layers.logits_processor import LogitsProcessor

36 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -37,7 +38,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

38 from vllm.model_executor.models.module_mapping import MultiModelKeys

39 from vllm.multimodal import MULTIMODAL_REGISTRY

40 from vllm.multimodal.inputs import (

40 - MultiModalDataDict,

41 MultiModalFieldConfig,

42 MultiModalKwargsItems,

43 )

vllm/model_executor/models/interfaces.py

+1 -2

@@ -30,8 +30,7 @@ from transformers.models.whisper.tokenization_whisper import LANGUAGES

30 from typing_extensions import Self, TypeIs

31

32 from vllm.config import ModelConfig, SpeechToTextConfig

33 -from vllm.inputs import TokensPrompt

34 -from vllm.inputs.data import PromptType

33 +from vllm.inputs import PromptType, TokensPrompt

34 from vllm.logger import init_logger

35 from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc

36 from vllm.model_executor.layers.quantization import QuantizationConfig

vllm/model_executor/models/interns1.py

+1 -1

@@ -23,12 +23,12 @@ from transformers.models.internvl.video_processing_internvl import (

23

24 from vllm.config import VllmConfig

25 from vllm.config.multimodal import BaseDummyOptions

26 +from vllm.inputs import MultiModalDataDict

27 from vllm.model_executor.layers.quantization import QuantizationConfig

28 from vllm.model_executor.models.interns1_vit import InternS1VisionModel

29 from vllm.model_executor.models.module_mapping import MultiModelKeys

30 from vllm.multimodal import MULTIMODAL_REGISTRY

31 from vllm.multimodal.inputs import (

31 - MultiModalDataDict,

32 MultiModalFieldConfig,

33 MultiModalKwargsItems,

34 )

vllm/model_executor/models/internvl.py

+1 -1

@@ -18,6 +18,7 @@ from transformers import BatchFeature, PretrainedConfig

18

19 from vllm.config import VllmConfig

20 from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.quantization import QuantizationConfig

23 from vllm.model_executor.layers.quantization.awq import AWQConfig

24 from vllm.model_executor.models.intern_vit import (

@@ -28,7 +29,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

29 from vllm.multimodal import MULTIMODAL_REGISTRY

30 from vllm.multimodal.inputs import (

31 BatchedTensorInputs,

31 - MultiModalDataDict,

32 MultiModalFieldConfig,

33 MultiModalKwargsItems,

34 )

vllm/model_executor/models/isaac.py

+1 -1

@@ -16,6 +16,7 @@ from vllm.config import ModelConfig, VllmConfig

16 from vllm.config.multimodal import BaseDummyOptions

17 from vllm.distributed import parallel_state

18 from vllm.distributed import utils as dist_utils

19 +from vllm.inputs import MultiModalDataDict

20 from vllm.model_executor.layers.attention import MMEncoderAttention

21 from vllm.model_executor.layers.linear import (

22 ColumnParallelLinear,

@@ -44,7 +45,6 @@ from vllm.model_executor.models.utils import (

45 )

46 from vllm.multimodal import MULTIMODAL_REGISTRY

47 from vllm.multimodal.inputs import (

47 - MultiModalDataDict,

48 MultiModalFeatureSpec,

49 MultiModalFieldConfig,

50 MultiModalKwargsItems,

vllm/model_executor/models/kanana_v.py

+1 -1

@@ -19,10 +19,10 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionCon

19

20 from vllm.config import VllmConfig

21 from vllm.config.multimodal import BaseDummyOptions

22 +from vllm.inputs import MultiModalDataDict

23 from vllm.logger import init_logger

24 from vllm.multimodal import MULTIMODAL_REGISTRY

25 from vllm.multimodal.inputs import (

25 - MultiModalDataDict,

26 MultiModalFieldConfig,

27 MultiModalKwargsItems,

28 )

vllm/model_executor/models/keye.py

+1 -2

@@ -19,6 +19,7 @@ from transformers.utils import torch_int

19 from vllm.config import VllmConfig

20 from vllm.config.multimodal import BaseDummyOptions

21 from vllm.distributed import get_tensor_model_parallel_world_size

22 +from vllm.inputs import ModalityData, MultiModalDataDict

23 from vllm.logger import init_logger

24 from vllm.model_executor.layers.attention import (

25 MMEncoderAttention,

@@ -41,8 +42,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

42 from vllm.multimodal import MULTIMODAL_REGISTRY

43 from vllm.multimodal.inputs import (

44 ImageItem,

44 - ModalityData,

45 - MultiModalDataDict,

45 MultiModalFeatureSpec,

46 MultiModalFieldConfig,

47 MultiModalKwargsItems,

vllm/model_executor/models/keye_vl1_5.py

+1 -1

@@ -14,13 +14,13 @@ from transformers.activations import GELUActivation

14 from transformers.feature_extraction_utils import BatchFeature

15

16 from vllm.config import VllmConfig

17 +from vllm.inputs import ModalityData

18 from vllm.logger import init_logger

19 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

20 from vllm.model_executor.layers.quantization import QuantizationConfig

21 from vllm.multimodal import MULTIMODAL_REGISTRY

22 from vllm.multimodal.inputs import (

23 ImageItem,

23 - ModalityData,

24 MultiModalFeatureSpec,

25 MultiModalFieldConfig,

26 MultiModalKwargsItems,

vllm/model_executor/models/kimi_audio.py

+1 -1

@@ -14,7 +14,7 @@ from transformers import WhisperConfig as HFWhisperConfig

14

15 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

16 from vllm.config.multimodal import BaseDummyOptions

17 -from vllm.inputs.data import PromptType, TokensPrompt

17 +from vllm.inputs import PromptType, TokensPrompt

18 from vllm.model_executor.model_loader import DefaultModelLoader

19 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

20 from vllm.model_executor.models.interfaces import (

vllm/model_executor/models/kimi_k25.py

+1 -1

@@ -16,6 +16,7 @@ from transformers import BatchFeature

16

17 from vllm.config import VllmConfig

18 from vllm.config.multimodal import BaseDummyOptions

19 +from vllm.inputs import MultiModalDataDict

20 from vllm.logger import init_logger

21 from vllm.model_executor.layers.quantization import QuantizationConfig

22 from vllm.model_executor.layers.quantization.compressed_tensors import (

@@ -35,7 +36,6 @@ from vllm.model_executor.models.kimi_k25_vit import (

36 )

37 from vllm.multimodal import MULTIMODAL_REGISTRY

38 from vllm.multimodal.inputs import (

38 - MultiModalDataDict,

39 MultiModalFieldConfig,

40 MultiModalKwargsItems,

41 NestedTensors,

vllm/model_executor/models/kimi_vl.py

+1 -1

@@ -54,12 +54,12 @@ from transformers.activations import GELUActivation

54

55 from vllm.config import VllmConfig

56 from vllm.config.multimodal import BaseDummyOptions

57 +from vllm.inputs import MultiModalDataDict

58 from vllm.model_executor.layers.linear import ReplicatedLinear

59 from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP

60 from vllm.model_executor.models.moonvit import MoonVitPretrainedModel

61 from vllm.multimodal import MULTIMODAL_REGISTRY

62 from vllm.multimodal.inputs import (

62 - MultiModalDataDict,

63 MultiModalFieldConfig,

64 MultiModalKwargsItems,

65 NestedTensors,

vllm/model_executor/models/lfm2_vl.py

+1 -1

@@ -21,6 +21,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import (

21 from vllm.config import VllmConfig

22 from vllm.config.multimodal import BaseDummyOptions

23 from vllm.forward_context import set_forward_context

24 +from vllm.inputs import MultiModalDataDict

25 from vllm.model_executor.layers.mamba.mamba_utils import (

26 MambaStateCopyFunc,

27 MambaStateCopyFuncCalculator,

@@ -30,7 +31,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (

31 from vllm.model_executor.models.module_mapping import MultiModelKeys

32 from vllm.multimodal import MULTIMODAL_REGISTRY

33 from vllm.multimodal.inputs import (

33 - MultiModalDataDict,

34 MultiModalFieldConfig,

35 MultiModalKwargsItems,

36 )

vllm/model_executor/models/llava.py

+3 -5

@@ -20,17 +20,15 @@ from transformers.models.pixtral import PixtralProcessor

20

21 from vllm.config import VllmConfig

22 from vllm.config.multimodal import BaseDummyOptions

23 +from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input

24 from vllm.model_executor.layers.activation import get_act_fn

25 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

26 from vllm.model_executor.layers.quantization import QuantizationConfig

27 from vllm.multimodal import MULTIMODAL_REGISTRY

28 from vllm.multimodal.cache import BaseMultiModalProcessorCache

29 from vllm.multimodal.inputs import (

29 - MultiModalDataDict,

30 MultiModalFieldConfig,

31 - MultiModalInputs,

31 MultiModalKwargsItems,

33 - mm_inputs,

32 )

33 from vllm.multimodal.parse import (

34 ImageEmbeddingItems,

@@ -777,7 +775,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

775 self,

776 inputs: ProcessorInputs,

777 timing_ctx: TimingContext,

780 - ) -> MultiModalInputs:

778 + ) -> MultiModalInput:

779 hf_config = self.info.get_hf_config()

780 image_token_id = hf_config.image_token_index

781

@@ -833,7 +831,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

831 for modality, placeholders in mm_placeholders.items()

832 }

833

836 - return mm_inputs(

834 + return mm_input(

835 prompt_token_ids=prompt_ids,

836 mm_kwargs=mm_kwargs,

837 mm_hashes=mm_hashes,

vllm/model_executor/models/llava_next_video.py

+1 -1

@@ -11,11 +11,11 @@ from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProce

11

12 from vllm.config import VllmConfig

13 from vllm.config.multimodal import BaseDummyOptions

14 +from vllm.inputs import MultiModalDataDict

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.models.clip import CLIPVisionModel

17 from vllm.multimodal import MULTIMODAL_REGISTRY

18 from vllm.multimodal.inputs import (

18 - MultiModalDataDict,

19 MultiModalFieldConfig,

20 MultiModalKwargsItems,

21 )

vllm/model_executor/models/llava_onevision.py

+1 -1

@@ -15,10 +15,10 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (

15

16 from vllm.config import VllmConfig

17 from vllm.config.multimodal import BaseDummyOptions

18 +from vllm.inputs import MultiModalDataDict

19 from vllm.model_executor.layers.activation import get_act_fn

20 from vllm.multimodal import MULTIMODAL_REGISTRY

21 from vllm.multimodal.inputs import (

21 - MultiModalDataDict,

22 MultiModalFieldConfig,

23 MultiModalKwargsItems,

24 )

vllm/model_executor/models/midashenglm.py

+1 -1

@@ -38,6 +38,7 @@ from transformers import BatchFeature

38 from vllm.config import VllmConfig

39 from vllm.config.multimodal import BaseDummyOptions

40 from vllm.distributed import get_tensor_model_parallel_world_size

41 +from vllm.inputs import MultiModalDataDict

42 from vllm.model_executor.layers.activation import get_act_fn

43 from vllm.model_executor.layers.conv import Conv2dLayer

44 from vllm.model_executor.layers.linear import (

@@ -48,7 +49,6 @@ from vllm.model_executor.layers.linear import (

49 from vllm.model_executor.layers.quantization import QuantizationConfig

50 from vllm.multimodal import MULTIMODAL_REGISTRY

51 from vllm.multimodal.inputs import (

51 - MultiModalDataDict,

52 MultiModalFieldConfig,

53 MultiModalKwargsItems,

54 )

vllm/model_executor/models/minicpmo.py

+1 -2

@@ -41,9 +41,9 @@ from transformers.models.whisper.modeling_whisper import (

41

42 from vllm.config import VllmConfig

43 from vllm.config.multimodal import BaseDummyOptions

44 +from vllm.inputs import ModalityData, MultiModalDataDict

45 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems

46 from vllm.multimodal.inputs import (

46 - MultiModalDataDict,

47 MultiModalFieldConfig,

48 NestedTensors,

49 )

@@ -51,7 +51,6 @@ from vllm.multimodal.parse import (

51 AudioItem,

52 AudioProcessorItems,

53 DictEmbeddingItems,

54 - ModalityData,

54 ModalityDataItems,

55 MultiModalDataItems,

56 )

vllm/model_executor/models/minicpmv.py

+1 -2

@@ -41,6 +41,7 @@ from typing_extensions import TypeVar

41

42 from vllm.config import VllmConfig

43 from vllm.config.multimodal import BaseDummyOptions

44 +from vllm.inputs import ModalityData, MultiModalDataDict

45 from vllm.model_executor.layers.quantization import QuantizationConfig

46 from vllm.model_executor.layers.resampler import (

47 BaseResampler,

@@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM

55 from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM

56 from vllm.multimodal import MULTIMODAL_REGISTRY

57 from vllm.multimodal.inputs import (

57 - MultiModalDataDict,

58 MultiModalFieldConfig,

59 MultiModalKwargsItems,

60 NestedTensors,

@@ -64,7 +64,6 @@ from vllm.multimodal.parse import (

64 ImageItem,

65 ImageProcessorItems,

66 ImageSize,

67 - ModalityData,

67 ModalityDataItems,

68 MultiModalDataItems,

69 MultiModalDataParser,

vllm/model_executor/models/mistral3.py

+1 -1

@@ -11,6 +11,7 @@ from transformers.models.pixtral import PixtralProcessor

11

12 from vllm.config import VllmConfig

13 from vllm.config.multimodal import BaseDummyOptions

14 +from vllm.inputs import MultiModalDataDict

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.layers.layernorm import RMSNorm

17 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -18,7 +19,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

19 from vllm.model_executor.models.module_mapping import MultiModelKeys

20 from vllm.multimodal import MULTIMODAL_REGISTRY

21 from vllm.multimodal.inputs import (

21 - MultiModalDataDict,

22 MultiModalFieldConfig,

23 MultiModalKwargsItems,

24 )

vllm/model_executor/models/mllama4.py

+1 -1

@@ -38,6 +38,7 @@ from vllm.compilation.decorators import (

38 from vllm.config import VllmConfig, set_current_vllm_config

39 from vllm.config.multimodal import BaseDummyOptions

40 from vllm.distributed import get_tensor_model_parallel_world_size

41 +from vllm.inputs import MultiModalDataDict

42 from vllm.model_executor.layers.attention import MMEncoderAttention

43 from vllm.model_executor.layers.fused_moe import FusedMoE

44 from vllm.model_executor.layers.linear import (

@@ -53,7 +54,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

54 from vllm.model_executor.models.module_mapping import MultiModelKeys

55 from vllm.multimodal import MULTIMODAL_REGISTRY

56 from vllm.multimodal.inputs import (

56 - MultiModalDataDict,

57 MultiModalFieldConfig,

58 MultiModalKwargsItems,

59 )

vllm/model_executor/models/molmo.py

+1 -1

@@ -29,6 +29,7 @@ from vllm.distributed import (

29 split_tensor_along_last_dim,

30 tensor_model_parallel_all_gather,

31 )

32 +from vllm.inputs import MultiModalDataDict

33 from vllm.model_executor.layers.activation import MulAndSilu, QuickGELU, SiluAndMul

34 from vllm.model_executor.layers.attention import Attention, MMEncoderAttention

35 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -49,7 +50,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

50 from vllm.model_executor.models.module_mapping import MultiModelKeys

51 from vllm.multimodal import MULTIMODAL_REGISTRY

52 from vllm.multimodal.inputs import (

52 - MultiModalDataDict,

53 MultiModalFieldConfig,

54 MultiModalKwargsItems,

55 )

vllm/model_executor/models/molmo2.py

+1 -1

@@ -33,6 +33,7 @@ from vllm.distributed import (

33 split_tensor_along_last_dim,

34 tensor_model_parallel_all_gather,

35 )

36 +from vllm.inputs import MultiModalDataDict

37 from vllm.logger import init_logger

38 from vllm.model_executor.layers.activation import MulAndSilu, SiluAndMul, get_act_fn

39 from vllm.model_executor.layers.attention import Attention, MMEncoderAttention

@@ -54,7 +55,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

55 from vllm.model_executor.models.module_mapping import MultiModelKeys

56 from vllm.multimodal import MULTIMODAL_REGISTRY

57 from vllm.multimodal.inputs import (

57 - MultiModalDataDict,

58 MultiModalFieldConfig,

59 MultiModalKwargsItems,

60 VideoItem,

vllm/model_executor/models/nano_nemotron_vl.py

+3 -4

@@ -21,6 +21,7 @@ from transformers import BatchFeature

21

22 from vllm.config import VllmConfig

23 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

24 +from vllm.inputs import MultiModalDataDict, MultiModalInput

25 from vllm.logger import init_logger

26 from vllm.model_executor.layers.activation import ReLUSquaredActivation

27 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -48,9 +49,7 @@ from vllm.multimodal.evs import (

49 from vllm.multimodal.inputs import (

50 AudioItem,

51 BatchedTensorInputs,

51 - MultiModalDataDict,

52 MultiModalFieldConfig,

53 - MultiModalInputs,

53 MultiModalKwargsItems,

54 VideoItem,

55 )

@@ -576,7 +575,7 @@ class NanoNemotronVLMultiModalProcessor(

575 self,

576 inputs: ProcessorInputs,

577 timing_ctx: TimingContext,

579 - ) -> MultiModalInputs:

578 + ) -> MultiModalInput:

579 use_audio_in_video = bool(

580 inputs.hf_processor_mm_kwargs.get("use_audio_in_video", False)

581 )

@@ -632,7 +631,7 @@ class NanoNemotronVLMultiModalProcessor(

631 for modality, placeholders in mm_placeholders.items()

632 }

633

635 - return MultiModalInputs(

634 + return MultiModalInput(

635 type="multimodal",

636 prompt_token_ids=prompt_ids,

637 mm_kwargs=mm_info.kwargs,

vllm/model_executor/models/nemotron_parse.py

+1 -1

@@ -23,6 +23,7 @@ from transformers import (

23 from vllm.config import CacheConfig, VllmConfig

24 from vllm.config.lora import LoRAConfig

25 from vllm.config.multimodal import BaseDummyOptions

26 +from vllm.inputs import MultiModalDataDict

27 from vllm.logger import init_logger

28 from vllm.model_executor.layers.activation import get_act_fn

29 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -41,7 +42,6 @@ from vllm.model_executor.models.radio import RadioModel

42 from vllm.model_executor.models.whisper import WhisperAttention, WhisperCrossAttention

43 from vllm.multimodal import MULTIMODAL_REGISTRY

44 from vllm.multimodal.inputs import (

44 - MultiModalDataDict,

45 MultiModalFieldConfig,

46 MultiModalKwargsItems,

47 )

vllm/model_executor/models/nvlm_d.py

+2 -4

@@ -14,12 +14,10 @@ import torch.nn as nn

14 from transformers import PretrainedConfig

15

16 from vllm.config.multimodal import BaseDummyOptions

17 +from vllm.inputs import MultiModalDataDict

18 from vllm.model_executor.layers.quantization import QuantizationConfig

19 from vllm.multimodal import MULTIMODAL_REGISTRY

19 -from vllm.multimodal.inputs import (

20 - BatchedTensorInputs,

21 - MultiModalDataDict,

22 -)

20 +from vllm.multimodal.inputs import BatchedTensorInputs

21 from vllm.multimodal.parse import (

22 ImageEmbeddingItems,

23 ImageProcessorItems,

vllm/model_executor/models/ovis.py

+1 -1

@@ -30,6 +30,7 @@ from transformers import BatchFeature, PretrainedConfig

30

31 from vllm.config import VllmConfig

32 from vllm.config.multimodal import BaseDummyOptions

33 +from vllm.inputs import MultiModalDataDict

34 from vllm.model_executor.layers.linear import ReplicatedLinear

35 from vllm.model_executor.layers.quantization import QuantizationConfig

36 from vllm.model_executor.models.aimv2 import AIMv2Model

@@ -42,7 +43,6 @@ from vllm.model_executor.models.utils import (

43 )

44 from vllm.multimodal import MULTIMODAL_REGISTRY

45 from vllm.multimodal.inputs import (

45 - MultiModalDataDict,

46 MultiModalFieldConfig,

47 MultiModalKwargsItems,

48 )

vllm/model_executor/models/ovis2_5.py

+1 -1

@@ -12,6 +12,7 @@ from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig

12

13 from vllm.config import VllmConfig

14 from vllm.config.multimodal import BaseDummyOptions

15 +from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.layers.linear import ReplicatedLinear

17 from vllm.model_executor.layers.quantization import QuantizationConfig

18 from vllm.model_executor.models.ovis import VisualEmbedding

@@ -24,7 +25,6 @@ from vllm.model_executor.models.utils import (

25 )

26 from vllm.multimodal import MULTIMODAL_REGISTRY

27 from vllm.multimodal.inputs import (

27 - MultiModalDataDict,

28 MultiModalFieldConfig,

29 MultiModalKwargsItems,

30 )

vllm/model_executor/models/paddleocr_vl.py

+1 -1

@@ -35,6 +35,7 @@ from vllm.config import VllmConfig

35 from vllm.config.multimodal import BaseDummyOptions

36 from vllm.distributed import parallel_state

37 from vllm.distributed import utils as dist_utils

38 +from vllm.inputs import MultiModalDataDict

39 from vllm.model_executor.layers.attention import (

40 MMEncoderAttention,

41 )

@@ -53,7 +54,6 @@ from vllm.model_executor.model_loader.weight_utils import (

54 )

55 from vllm.multimodal import MULTIMODAL_REGISTRY

56 from vllm.multimodal.inputs import (

56 - MultiModalDataDict,

57 MultiModalFeatureSpec,

58 MultiModalFieldConfig,

59 MultiModalKwargsItems,

vllm/model_executor/models/paligemma.py

+2 -3

@@ -9,12 +9,11 @@ from transformers import BatchFeature, PaliGemmaConfig

9

10 from vllm.config import VllmConfig

11 from vllm.config.multimodal import BaseDummyOptions

12 +from vllm.inputs import MultiModalDataDict, MultiModalInput

13 from vllm.logger import init_logger

14 from vllm.multimodal import MULTIMODAL_REGISTRY

15 from vllm.multimodal.inputs import (

15 - MultiModalDataDict,

16 MultiModalFieldConfig,

17 - MultiModalInputs,

17 MultiModalKwargsItems,

18 )

19 from vllm.multimodal.parse import (

@@ -231,7 +230,7 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn

230 self,

231 inputs: ProcessorInputs,

232 timing_ctx: TimingContext,

234 - ) -> MultiModalInputs:

233 + ) -> MultiModalInput:

234 mm_inputs = super().apply(inputs, timing_ctx)

235 prompt_token_ids = mm_inputs["prompt_token_ids"]

236

vllm/model_executor/models/phi3v.py

+1 -1

@@ -30,12 +30,12 @@ from transformers import (

30

31 from vllm.config import VllmConfig

32 from vllm.config.multimodal import BaseDummyOptions

33 +from vllm.inputs import MultiModalDataDict

34 from vllm.logger import init_logger

35 from vllm.model_executor.layers.quantization import QuantizationConfig

36 from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding

37 from vllm.multimodal import MULTIMODAL_REGISTRY

38 from vllm.multimodal.inputs import (

38 - MultiModalDataDict,

39 MultiModalFieldConfig,

40 MultiModalKwargsItems,

41 )

vllm/model_executor/models/phi4mm.py

+1 -1

@@ -18,6 +18,7 @@ from transformers import (

18 from vllm.config import VllmConfig

19 from vllm.config.multimodal import BaseDummyOptions

20 from vllm.distributed import get_pp_group

21 +from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.logits_processor import LogitsProcessor

23 from vllm.model_executor.layers.quantization import QuantizationConfig

24 from vllm.model_executor.layers.vocab_parallel_embedding import (

@@ -27,7 +28,6 @@ from vllm.model_executor.models.llama import LlamaModel

28 from vllm.model_executor.models.module_mapping import MultiModelKeys

29 from vllm.multimodal import MULTIMODAL_REGISTRY

30 from vllm.multimodal.inputs import (

30 - MultiModalDataDict,

31 MultiModalFieldConfig,

32 MultiModalKwargsItems,

33 NestedTensors,

vllm/model_executor/models/pixtral.py

+1 -1

@@ -25,6 +25,7 @@ from transformers.models.pixtral.modeling_pixtral import (

25 from vllm.config import VllmConfig

26 from vllm.config.multimodal import BaseDummyOptions

27 from vllm.distributed import divide, get_tensor_model_parallel_world_size

28 +from vllm.inputs import MultiModalDataDict

29 from vllm.model_executor.layers.activation import get_act_and_mul_fn

30 from vllm.model_executor.layers.conv import Conv2dLayer

31 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -37,7 +38,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

39 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems

40 from vllm.multimodal.inputs import (

40 - MultiModalDataDict,

41 MultiModalFieldConfig,

42 NestedTensors,

43 )

vllm/model_executor/models/qwen2_5_omni_thinker.py

+1 -2

@@ -46,6 +46,7 @@ from transformers.models.whisper import WhisperFeatureExtractor

46 from vllm.config import VllmConfig

47 from vllm.config.multimodal import BaseDummyOptions

48 from vllm.forward_context import set_forward_context

49 +from vllm.inputs import ModalityData, MultiModalDataDict

50 from vllm.logger import init_logger

51 from vllm.model_executor.models.module_mapping import MultiModelKeys

52 from vllm.model_executor.models.qwen2_5_vl import (

@@ -66,8 +67,6 @@ from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser

67 from vllm.multimodal import MULTIMODAL_REGISTRY

68 from vllm.multimodal.inputs import (

69 ImageItem,

69 - ModalityData,

70 - MultiModalDataDict,

70 MultiModalFeatureSpec,

71 MultiModalFieldConfig,

72 MultiModalKwargsItems,

vllm/model_executor/models/qwen2_audio.py

+1 -2

@@ -38,11 +38,10 @@ from transformers.models.whisper import WhisperFeatureExtractor

38

39 from vllm.config import VllmConfig

40 from vllm.config.multimodal import BaseDummyOptions

41 +from vllm.inputs import ModalityData, MultiModalDataDict

42 from vllm.multimodal import MULTIMODAL_REGISTRY

43 from vllm.multimodal.inputs import (

44 AudioItem,

44 - ModalityData,

45 - MultiModalDataDict,

45 MultiModalFieldConfig,

46 MultiModalKwargsItems,

47 )

vllm/model_executor/models/qwen2_vl.py

+1 -2

@@ -47,6 +47,7 @@ from vllm.config import VllmConfig

47 from vllm.config.multimodal import BaseDummyOptions

48 from vllm.distributed import parallel_state, tensor_model_parallel_all_gather

49 from vllm.distributed import utils as dist_utils

50 +from vllm.inputs import ModalityData, MultiModalDataDict

51 from vllm.logger import init_logger

52 from vllm.model_executor.layers.activation import QuickGELU

53 from vllm.model_executor.layers.attention import MMEncoderAttention

@@ -65,8 +66,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

66 from vllm.multimodal import MULTIMODAL_REGISTRY

67 from vllm.multimodal.inputs import (

68 ImageItem,

68 - ModalityData,

69 - MultiModalDataDict,

69 MultiModalFeatureSpec,

70 MultiModalFieldConfig,

71 MultiModalKwargsItems,

vllm/model_executor/models/qwen3_asr.py

+1 -3

@@ -33,7 +33,7 @@ from transformers.models.whisper import WhisperFeatureExtractor

33

34 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

35 from vllm.config.multimodal import BaseDummyOptions

36 -from vllm.inputs.data import PromptType, TokensPrompt

36 +from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt

37 from vllm.logger import init_logger

38 from vllm.model_executor.models.interfaces import (

39 MultiModalEmbeddings,

@@ -59,8 +59,6 @@ from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS

59 from vllm.multimodal import MULTIMODAL_REGISTRY

60 from vllm.multimodal.inputs import (

61 AudioItem,

62 - ModalityData,

63 - MultiModalDataDict,

62 MultiModalFeatureSpec,

63 MultiModalFieldConfig,

64 MultiModalKwargsItems,

vllm/model_executor/models/qwen3_asr_realtime.py

+1 -1

@@ -23,7 +23,7 @@ import numpy as np

23 import torch

24

25 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

26 -from vllm.inputs.data import PromptType, TokensPrompt

26 +from vllm.inputs import PromptType, TokensPrompt

27 from vllm.logger import init_logger

28 from vllm.model_executor.models.interfaces import (

29 SupportsRealtime,

vllm/model_executor/models/qwen3_omni_moe_thinker.py

+3 -6

@@ -31,6 +31,8 @@ import torch

31 import torch.nn as nn

32 import torch.nn.functional as F

33 from packaging.version import Version

34 +from transformers import PretrainedConfig

35 +from transformers import __version__ as TRANSFORMERS_VERSION

36 from transformers.feature_extraction_utils import BatchFeature

37 from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import (

38 Qwen3OmniMoeAudioEncoderConfig,

@@ -42,15 +44,10 @@ from transformers.models.qwen3_omni_moe.processing_qwen3_omni_moe import (

44 )

45 from transformers.models.whisper import WhisperFeatureExtractor

46

45 -# isort: off

46 -from transformers import PretrainedConfig

47 -from transformers import __version__ as TRANSFORMERS_VERSION

48 -# isort: on

49 -

47 from vllm.compilation.decorators import support_torch_compile

48 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

49 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size

53 -from vllm.inputs.data import PromptType

50 +from vllm.inputs import PromptType

51 from vllm.logger import init_logger

52 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

53 from vllm.model_executor.layers.attention.mm_encoder_attention import (

vllm/model_executor/models/qwen3_vl.py

+1 -1

@@ -52,6 +52,7 @@ from vllm.compilation.decorators import support_torch_compile

52 from vllm.config import VllmConfig

53 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

54 from vllm.distributed import get_pp_group, parallel_state

55 +from vllm.inputs import MultiModalDataDict

56 from vllm.logger import init_logger

57 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

58 from vllm.model_executor.layers.attention.mm_encoder_attention import (

@@ -76,7 +77,6 @@ from vllm.multimodal.evs import (

77 recompute_mrope_positions,

78 )

79 from vllm.multimodal.inputs import (

79 - MultiModalDataDict,

80 MultiModalFeatureSpec,

81 MultiModalFieldConfig,

82 MultiModalFieldElem,

vllm/model_executor/models/qwen_vl.py

+1 -1

@@ -18,6 +18,7 @@ from transformers import BatchFeature

18

19 from vllm.config import VllmConfig

20 from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.activation import get_act_fn

23 from vllm.model_executor.layers.conv import Conv2dLayer

24 from vllm.model_executor.layers.linear import (

@@ -30,7 +31,6 @@ from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos

31 from vllm.model_executor.models.module_mapping import MultiModelKeys

32 from vllm.multimodal import MULTIMODAL_REGISTRY

33 from vllm.multimodal.inputs import (

33 - MultiModalDataDict,

34 MultiModalFieldConfig,

35 MultiModalKwargsItems,

36 )

vllm/model_executor/models/rvl.py

+1 -1

@@ -9,8 +9,8 @@ from transformers.activations import GELUActivation

9

10 from vllm.config import VllmConfig

11 from vllm.config.multimodal import BaseDummyOptions

12 +from vllm.inputs import MultiModalDataDict

13 from vllm.multimodal import MULTIMODAL_REGISTRY

13 -from vllm.multimodal.inputs import MultiModalDataDict

14

15 from .llava_next import (

16 LlavaDummyInputsBuilder,

vllm/model_executor/models/siglip.py

+2 -3

@@ -18,6 +18,7 @@ from transformers import (

18 from vllm.config import VllmConfig

19 from vllm.config.multimodal import BaseDummyOptions

20 from vllm.distributed import divide, get_tensor_model_parallel_world_size

21 +from vllm.inputs import MultiModalDataDict, MultiModalInput

22 from vllm.model_executor.layers.activation import get_act_fn

23 from vllm.model_executor.layers.attention import (

24 EncoderOnlyAttention,

@@ -38,9 +39,7 @@ from vllm.model_executor.model_loader.weight_utils import (

39 )

40 from vllm.multimodal import MULTIMODAL_REGISTRY

41 from vllm.multimodal.inputs import (

41 - MultiModalDataDict,

42 MultiModalFieldConfig,

43 - MultiModalInputs,

43 MultiModalKwargsItems,

44 )

45 from vllm.multimodal.parse import (

@@ -193,7 +192,7 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):

192 self,

193 inputs: ProcessorInputs,

194 timing_ctx: TimingContext,

196 - ) -> MultiModalInputs:

195 + ) -> MultiModalInput:

196 if inputs.mm_data_items:

197 if isinstance(inputs.prompt, str):

198 if len(inputs.prompt) > 0:

vllm/model_executor/models/skyworkr1v.py

+1 -1

@@ -16,6 +16,7 @@ from transformers import PretrainedConfig

16

17 from vllm.config import VllmConfig

18 from vllm.config.multimodal import BaseDummyOptions

19 +from vllm.inputs import MultiModalDataDict

20 from vllm.model_executor.layers.linear import ReplicatedLinear

21 from vllm.model_executor.layers.quantization import QuantizationConfig

22 from vllm.model_executor.layers.quantization.awq import AWQConfig

@@ -24,7 +25,6 @@ from vllm.model_executor.models.intern_vit import (

25 InternVisionPatchModel,

26 )

27 from vllm.multimodal import MULTIMODAL_REGISTRY

27 -from vllm.multimodal.inputs import MultiModalDataDict

28 from vllm.multimodal.processing import BaseDummyInputsBuilder

29 from vllm.sequence import IntermediateTensors

30 from vllm.transformers_utils.processors.internvl import (

vllm/model_executor/models/step3_vl.py

+1 -1

@@ -13,6 +13,7 @@ from transformers import BatchFeature

13 from vllm.config import VllmConfig

14 from vllm.config.multimodal import BaseDummyOptions

15 from vllm.distributed import get_tensor_model_parallel_world_size

16 +from vllm.inputs import MultiModalDataDict

17 from vllm.model_executor.layers.activation import get_act_fn

18 from vllm.model_executor.layers.attention import MMEncoderAttention

19 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -24,7 +25,6 @@ from vllm.model_executor.layers.linear import (

25 from vllm.model_executor.layers.quantization import QuantizationConfig

26 from vllm.multimodal import MULTIMODAL_REGISTRY

27 from vllm.multimodal.inputs import (

27 - MultiModalDataDict,

28 MultiModalFieldConfig,

29 MultiModalKwargsItems,

30 )

vllm/model_executor/models/terratorch.py

+3 -6

@@ -34,6 +34,7 @@ from transformers import BatchFeature

34

35 from vllm.config import VllmConfig

36 from vllm.config.multimodal import BaseDummyOptions

37 +from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalInput, mm_input

38 from vllm.logger import init_logger

39 from vllm.model_executor.layers.pooler import IdentityPooler

40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -41,13 +42,9 @@ from vllm.model_executor.models.utils import AutoWeightsLoader

42 from vllm.multimodal import MULTIMODAL_REGISTRY

43 from vllm.multimodal.inputs import (

44 ImageItem,

44 - ModalityData,

45 - MultiModalDataDict,

45 MultiModalFieldConfig,

47 - MultiModalInputs,

46 MultiModalKwargsItems,

47 PlaceholderRange,

50 - mm_inputs,

48 )

49 from vllm.multimodal.parse import (

50 DictEmbeddingItems,

@@ -196,7 +193,7 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing

193 self,

194 inputs: ProcessorInputs,

195 timing_ctx: TimingContext,

199 - ) -> MultiModalInputs:

196 + ) -> MultiModalInput:

197 mm_items = inputs.mm_data_items

198 hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs

199

@@ -224,7 +221,7 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing

221

222 mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}

223

227 - return mm_inputs(

224 + return mm_input(

225 prompt_token_ids=[1],

226 mm_kwargs=mm_kwargs,

227 mm_hashes=mm_hashes,

vllm/model_executor/models/transformers/multimodal.py

+3 -5

@@ -22,16 +22,14 @@ from typing import TYPE_CHECKING

22 import torch

23

24 from vllm.config.utils import getattr_iter

25 +from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input

26 from vllm.logger import init_logger

27 from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal

28 from vllm.multimodal import MultiModalKwargsItems

29 from vllm.multimodal.inputs import (

29 - MultiModalDataDict,

30 MultiModalFeatureSpec,

31 MultiModalFieldConfig,

32 - MultiModalInputs,

32 PlaceholderRange,

34 - mm_inputs,

33 )

34 from vllm.multimodal.parse import (

35 ImageProcessorItems,

@@ -179,7 +177,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):

177 self,

178 inputs: ProcessorInputs,

179 timing_ctx: TimingContext,

182 - ) -> MultiModalInputs:

180 + ) -> MultiModalInput:

181 """

182 Process multi-modal inputs to be used in vLLM.

183

@@ -261,7 +259,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):

259 with timing_ctx.record("get_mm_hashes"):

260 mm_hashes = inputs.get_mm_hashes(self.info.model_id)

261

264 - return mm_inputs(

262 + return mm_input(

263 prompt_token_ids=prompt_ids,

264 mm_kwargs=mm_kwargs,

265 mm_hashes=mm_hashes,

vllm/model_executor/models/ultravox.py

+1 -1

@@ -23,13 +23,13 @@ from transformers.models.whisper.modeling_whisper import (

23

24 from vllm.config import VllmConfig

25 from vllm.config.multimodal import BaseDummyOptions

26 +from vllm.inputs import MultiModalDataDict

27 from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn

28 from vllm.model_executor.layers.layernorm import RMSNorm

29 from vllm.model_executor.model_loader import DefaultModelLoader

30 from vllm.model_executor.models.module_mapping import MultiModelKeys

31 from vllm.multimodal import MULTIMODAL_REGISTRY

32 from vllm.multimodal.inputs import (

32 - MultiModalDataDict,

33 MultiModalFieldConfig,

34 MultiModalKwargsItems,

35 NestedTensors,

vllm/model_executor/models/voxtral.py

+1 -2

@@ -19,7 +19,7 @@ from transformers import BatchFeature, WhisperConfig

19

20 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

21 from vllm.config.multimodal import BaseDummyOptions

22 -from vllm.inputs.data import PromptType, TokensPrompt

22 +from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt

23 from vllm.logger import init_logger

24 from vllm.model_executor.layers.quantization import QuantizationConfig

25 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -32,7 +32,6 @@ from vllm.model_executor.models.whisper import (

32 from vllm.model_executor.models.whisper_causal import WhisperCausalEncoder

33 from vllm.multimodal import MULTIMODAL_REGISTRY

34 from vllm.multimodal.inputs import (

35 - MultiModalDataDict,

35 MultiModalFieldConfig,

36 MultiModalKwargsItems,

37 NestedTensors,

vllm/model_executor/models/voxtral_realtime.py

+2 -4

@@ -20,7 +20,7 @@ from vllm.compilation.decorators import support_torch_compile

20 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

21 from vllm.engine.protocol import StreamingInput

22 from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S

23 -from vllm.inputs.data import PromptType, TokensPrompt

23 +from vllm.inputs import PromptType, TokensPrompt

24 from vllm.logger import init_logger

25 from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime

26 from vllm.model_executor.models.voxtral import (

@@ -31,9 +31,7 @@ from vllm.model_executor.models.voxtral import (

31 )

32 from vllm.multimodal import MULTIMODAL_REGISTRY

33 from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache

34 -from vllm.multimodal.inputs import (

35 - MultiModalKwargsOptionalItems,

36 -)

34 +from vllm.multimodal.inputs import MultiModalKwargsOptionalItems

35 from vllm.multimodal.parse import MultiModalDataItems

36 from vllm.multimodal.processing import BaseDummyInputsBuilder

37 from vllm.multimodal.processing.processor import (

vllm/model_executor/models/whisper.py

+6 -2

@@ -21,7 +21,12 @@ from vllm.compilation.decorators import support_torch_compile

21 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

22 from vllm.config.multimodal import BaseDummyOptions

23 from vllm.distributed import get_tensor_model_parallel_world_size

24 -from vllm.inputs.data import ExplicitEncoderDecoderPrompt, PromptType, TextPrompt

24 +from vllm.inputs import (

25 + ExplicitEncoderDecoderPrompt,

26 + MultiModalDataDict,

27 + PromptType,

28 + TextPrompt,

29 +)

30 from vllm.logger import init_logger

31 from vllm.model_executor.layers.activation import get_act_fn

32 from vllm.model_executor.layers.attention import (

@@ -44,7 +49,6 @@ from vllm.model_executor.models.whisper_utils import (

49 )

50 from vllm.multimodal import MULTIMODAL_REGISTRY

51 from vllm.multimodal.inputs import (

47 - MultiModalDataDict,

52 MultiModalFieldConfig,

53 MultiModalKwargsItems,

54 )

vllm/multimodal/__init__.py

+1 -15

@@ -1,16 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 from .hasher import MultiModalHasher

4 -from .inputs import (

5 - BatchedTensorInputs,

6 - ModalityData,

7 - MultiModalDataBuiltins,

8 - MultiModalDataDict,

9 - MultiModalKwargsItems,

10 - MultiModalPlaceholderDict,

11 - MultiModalUUIDDict,

12 - NestedTensors,

13 -)

4 +from .inputs import BatchedTensorInputs, MultiModalKwargsItems, NestedTensors

5 from .registry import MultiModalRegistry

6

7 MULTIMODAL_REGISTRY = MultiModalRegistry()

@@ -25,13 +16,8 @@ Info:

16

17 __all__ = [

18 "BatchedTensorInputs",

28 - "ModalityData",

29 - "MultiModalDataBuiltins",

30 - "MultiModalDataDict",

19 "MultiModalHasher",

20 "MultiModalKwargsItems",

33 - "MultiModalPlaceholderDict",

34 - "MultiModalUUIDDict",

21 "NestedTensors",

22 "MULTIMODAL_REGISTRY",

23 "MultiModalRegistry",

vllm/multimodal/inputs.py

+2 -162

@@ -15,12 +15,11 @@ from typing import (

15 TypedDict,

16 Union,

17 cast,

18 - final,

18 )

19

20 import numpy as np

21 from PIL.Image import Image

23 -from typing_extensions import NotRequired, TypeVar

22 +from typing_extensions import TypeVar

23

24 from vllm.utils.collection_utils import is_list_of

25 from vllm.utils.import_utils import LazyLoader

@@ -32,14 +31,9 @@ if TYPE_CHECKING:

31 import torch

32 import torch.types

33 from transformers.feature_extraction_utils import BatchFeature

35 -

36 - from vllm.inputs.data import _InputOptions

34 else:

35 torch = LazyLoader("torch", globals(), "torch")

36

40 - _InputOptions = dict

41 -

42 -_T = TypeVar("_T")

37

38 HfImageItem: TypeAlias = Union["Image", np.ndarray, "torch.Tensor"]

39 """

@@ -98,15 +92,6 @@ which are treated as audio embeddings;

92 these are directly passed to the model without HF processing.

93 """

94

101 -ModalityData: TypeAlias = _T | list[_T | None] | None

102 -"""

103 -Either a single data item, or a list of data items. Can only be None if UUID

104 -is provided.

105 -

106 -The number of data items allowed per modality is restricted by

107 -`--limit-mm-per-prompt`.

108 -"""

109 -

95

96 class VisionChunkImage(TypedDict):

97 """Represents an image wrapped as a vision chunk."""

@@ -126,46 +111,10 @@ class VisionChunkVideo(TypedDict):

111 video_idx: int

112

113

129 -VisionChunk = VisionChunkImage | VisionChunkVideo

114 +VisionChunk: TypeAlias = VisionChunkImage | VisionChunkVideo

115 """A vision chunk is either an image or a video chunk."""

116

117

133 -@final

134 -class MultiModalDataBuiltins(TypedDict, total=False):

135 - """Type annotations for modality types predefined by vLLM."""

136 -

137 - image: ModalityData[ImageItem]

138 - """The input image(s)."""

139 -

140 - video: ModalityData[VideoItem]

141 - """The input video(s)."""

142 -

143 - audio: ModalityData[AudioItem]

144 - """The input audio(s)."""

145 -

146 - vision_chunk: ModalityData[VisionChunk]

147 - """The input visual atom(s) - unified modality for images and video chunks."""

148 -

149 -

150 -MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]

151 -"""

152 -A dictionary containing an entry for each modality type to input.

153 -

154 -The built-in modalities are defined by

155 -[`MultiModalDataBuiltins`][vllm.multimodal.inputs.MultiModalDataBuiltins].

156 -"""

157 -

158 -MultiModalUUIDDict: TypeAlias = Mapping[str, Sequence[str | None] | str]

159 -"""

160 -A dictionary containing user-provided UUIDs for items in each modality.

161 -If a UUID for an item is not provided, its entry will be `None` and

162 -MultiModalHasher will compute a hash for the item.

163 -

164 -The UUID will be used to identify the item for all caching purposes

165 -(input processing caching, embedding caching, prefix caching, etc).

166 -"""

167 -

168 -

118 @dataclass(frozen=True)

119 class PlaceholderRange:

120 """

@@ -1048,112 +997,3 @@ MultiModalKwargsOptionalItems: TypeAlias = (

997 MultiModalKwargsItems[MultiModalKwargsItem]

998 | MultiModalKwargsItems[MultiModalKwargsItem | None]

999 )

1051 -

1052 -

1053 -MultiModalHashes = dict[str, list[str]]

1054 -"""

1055 -A dictionary containing per-item hashes for each modality.

1056 -"""

1057 -

1058 -

1059 -MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]]

1060 -"""

1061 -A dictionary containing per-item placeholder ranges for each modality.

1062 -"""

1063 -

1064 -

1065 -class MultiModalInputs(_InputOptions):

1066 - """

1067 - Represents the outputs of

1068 - [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor],

1069 - ready to be passed to vLLM internals.

1070 - """

1071 -

1072 - type: Literal["multimodal"]

1073 - """The type of inputs."""

1074 -

1075 - prompt_token_ids: list[int]

1076 - """The processed token IDs which includes placeholder tokens."""

1077 -

1078 - prompt: NotRequired[str]

1079 - """The prompt text corresponding to the token IDs, if available."""

1080 -

1081 - mm_kwargs: MultiModalKwargsOptionalItems

1082 - """Keyword arguments to be directly passed to the model after batching."""

1083 -

1084 - mm_hashes: MultiModalHashes

1085 - """The hashes of the multi-modal data."""

1086 -

1087 - mm_placeholders: MultiModalPlaceholderDict

1088 - """

1089 - For each modality, information about the placeholder tokens in

1090 - `prompt_token_ids`.

1091 - """

1092 -

1093 -

1094 -def mm_inputs(

1095 - prompt_token_ids: list[int],

1096 - mm_kwargs: MultiModalKwargsOptionalItems,

1097 - mm_hashes: MultiModalHashes,

1098 - mm_placeholders: MultiModalPlaceholderDict,

1099 - *,

1100 - prompt: str | None = None,

1101 - cache_salt: str | None = None,

1102 -) -> MultiModalInputs:

1103 - inputs = MultiModalInputs(

1104 - type="multimodal",

1105 - prompt_token_ids=prompt_token_ids,

1106 - mm_kwargs=mm_kwargs,

1107 - mm_hashes=mm_hashes,

1108 - mm_placeholders=mm_placeholders,

1109 - )

1110 -

1111 - if prompt is not None:

1112 - inputs["prompt"] = prompt

1113 - if cache_salt is not None:

1114 - inputs["cache_salt"] = cache_salt

1115 -

1116 - return inputs

1117 -

1118 -

1119 -class MultiModalEncDecInputs(MultiModalInputs):

1120 - """

1121 - Represents the outputs of

1122 - [`EncDecMultiModalProcessor`][vllm.multimodal.processing.EncDecMultiModalProcessor]

1123 - ready to be passed to vLLM internals.

1124 -

1125 - Note: Even text-only encoder-decoder models are currently implemented

1126 - as multi-modal models for convenience.

1127 - (Example: https://github.com/vllm-project/bart-plugin)

1128 - """

1129 -

1130 - encoder_prompt_token_ids: list[int]

1131 - """The processed token IDs of the encoder prompt."""

1132 -

1133 - encoder_prompt: NotRequired[str]

1134 - """The prompt text corresponding to the encoder token IDs, if available."""

1135 -

1136 -

1137 -def mm_enc_dec_inputs(

1138 - encoder_inputs: MultiModalInputs,

1139 - decoder_prompt_token_ids: list[int],

1140 - *,

1141 - decoder_prompt: str | None = None,

1142 -) -> MultiModalEncDecInputs:

1143 - inputs = MultiModalEncDecInputs(

1144 - type="multimodal",

1145 - prompt_token_ids=decoder_prompt_token_ids,

1146 - encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],

1147 - mm_kwargs=encoder_inputs["mm_kwargs"],

1148 - mm_hashes=encoder_inputs["mm_hashes"],

1149 - mm_placeholders=encoder_inputs["mm_placeholders"],

1150 - )

1151 -

1152 - if decoder_prompt is not None:

1153 - inputs["prompt"] = decoder_prompt

1154 - if "prompt" in encoder_inputs:

1155 - inputs["encoder_prompt"] = encoder_inputs["prompt"]

1156 - if "cache_salt" in encoder_inputs:

1157 - inputs["cache_salt"] = encoder_inputs["cache_salt"]

1158 -

1159 - return inputs

vllm/multimodal/parse.py

+6 -8

@@ -19,6 +19,7 @@ import numpy as np

19 import torch

20 from typing_extensions import assert_never

21

22 +from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalUUIDDict

23 from vllm.utils.collection_utils import is_list_of

24 from vllm.utils.import_utils import LazyLoader

25

@@ -29,11 +30,8 @@ from .inputs import (

30 HfImageItem,

31 HfVideoItem,

32 ImageItem,

32 - ModalityData,

33 - MultiModalDataDict,

33 MultiModalFieldConfig,

34 MultiModalKwargsItems,

36 - MultiModalUUIDDict,

35 VideoItem,

36 )

37 from .media import MediaWithBytes

@@ -407,8 +405,8 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])

405

406 class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):

407 """

410 - As [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict], but

411 - normalized such that each entry corresponds to a list.

408 + A normalized [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]

409 + such that each entry corresponds to a list.

410 """

411

412 def select(self, modalities: Set[str]):

@@ -477,7 +475,7 @@ ModalityDataParser: TypeAlias = Callable[

475

476 class MultiModalDataParser:

477 """

480 - Parses [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]

478 + Parses [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]

479 into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].

480

481 Args:

@@ -695,8 +693,8 @@ class MultiModalDataParser:

693

694 MultiModalUUIDItems: TypeAlias = dict[str, Sequence[str | None]]

695 """

698 -As [`MultiModalUUIDDict`][vllm.multimodal.inputs.MultiModalUUIDDict], but

699 -normalized such that each entry corresponds to a list.

696 +A normalized [`MultiModalUUIDDict`][vllm.inputs.MultiModalUUIDDict]

697 +such that each entry corresponds to a list.

698 """

699

700

vllm/multimodal/processing/context.py

+6 -4

@@ -11,15 +11,14 @@ from typing import TYPE_CHECKING, Any, overload

11 import torch

12 from typing_extensions import TypeVar

13

14 +from vllm.inputs import MultiModalDataDict

15 from vllm.logger import init_logger

15 -from vllm.multimodal.inputs import MultiModalDataDict

16 from vllm.multimodal.parse import (

17 DictEmbeddingItems,

18 EmbeddingItems,

19 MultiModalDataItems,

20 MultiModalDataParser,

21 )

22 -from vllm.renderers import TokenizeParams

22 from vllm.tokenizers import TokenizerLike

23 from vllm.transformers_utils.processor import cached_processor_from_config

24 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

@@ -32,12 +31,14 @@ if TYPE_CHECKING:

31 from transformers.processing_utils import ProcessorMixin

32

33 from vllm.config import ModelConfig

34 + from vllm.renderers import TokenizeParams

35 else:

36 PretrainedConfig = object

37 BatchFeature = object

38 ProcessorMixin = object

39

40 ModelConfig = object

41 + TokenizeParams = object

42

43 logger = init_logger(__name__)

44

@@ -339,6 +340,8 @@ class BaseProcessingInfo:

340

341 def get_default_tok_params(self) -> TokenizeParams:

342 """Construct the default parameters for tokenization."""

343 + from vllm.renderers import TokenizeParams

344 +

345 model_config = self.ctx.model_config

346 encoder_config = model_config.encoder_config or {}

347

@@ -451,8 +454,7 @@ class BaseProcessingInfo:

454 validate: bool = True,

455 ) -> MultiModalDataItems:

456 """

454 - Normalize

455 - [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]

457 + Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]

458 to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]

459 before passing them to

460 [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].

vllm/multimodal/processing/dummy_inputs.py

+1 -1

@@ -14,9 +14,9 @@ from vllm.config.multimodal import (

14 ImageDummyOptions,

15 VideoDummyOptions,

16 )

17 +from vllm.inputs import MultiModalDataDict

18 from vllm.logger import init_logger

19

19 -from ..inputs import MultiModalDataDict

20 from .context import BaseProcessingInfo

21 from .inputs import ProcessorInputs

22

vllm/multimodal/processing/inputs.py

+3 -2

@@ -3,8 +3,9 @@

3 from collections.abc import Mapping

4 from dataclasses import dataclass, field

5

6 +from vllm.inputs import MultiModalHashes

7 +

8 from ..hasher import MultiModalHasher

7 -from ..inputs import MultiModalHashes

9 from ..parse import MultiModalDataItems, MultiModalUUIDItems

10

11

@@ -26,7 +27,7 @@ class ProcessorInputs:

27 mm_uuid_items = self.mm_uuid_items or {}

28 hf_processor_mm_kwargs = self.hf_processor_mm_kwargs

29

29 - mm_hashes: MultiModalHashes = {}

30 + mm_hashes = dict[str, list[str]]()

31 hasher = MultiModalHasher

32

33 for modality, data_items in mm_data_items.items():

vllm/multimodal/processing/processor.py

+14 -19

@@ -6,34 +6,29 @@ from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, S

6 from dataclasses import dataclass, field, replace

7 from enum import Enum

8 from functools import lru_cache

9 -from typing import (

10 - TYPE_CHECKING,

11 - Generic,

12 - NamedTuple,

13 - Protocol,

14 - TypeAlias,

15 - cast,

16 -)

9 +from typing import TYPE_CHECKING, Generic, NamedTuple, Protocol, TypeAlias, cast

10

11 import regex as re

12 import torch

13 from typing_extensions import TypeVar, assert_never

14

15 +from vllm.inputs import (

16 + MultiModalEncDecInput,

17 + MultiModalHashes,

18 + MultiModalInput,

19 + mm_enc_dec_input,

20 + mm_input,

21 +)

22 from vllm.logger import init_logger

23 from vllm.tokenizers import TokenizerLike

24 from vllm.utils.collection_utils import flatten_2d_lists, full_groupby

25

26 from ..inputs import (

27 - MultiModalEncDecInputs,

27 MultiModalFieldConfig,

29 - MultiModalHashes,

30 - MultiModalInputs,

28 MultiModalKwargsItem,

29 MultiModalKwargsItems,

30 MultiModalKwargsOptionalItems,

31 PlaceholderRange,

35 - mm_enc_dec_inputs,

36 - mm_inputs,

32 )

33 from ..parse import (

34 DictEmbeddingItems,

@@ -994,7 +989,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

989 mm_items: MultiModalDataItems,

990 mm_uuid_items: MultiModalUUIDItems | None = None,

991 hf_processor_mm_kwargs: Mapping[str, object] | None = None,

997 - ) -> MultiModalInputs:

992 + ) -> MultiModalInput:

993 processor_inputs = ProcessorInputs(

994 prompt,

995 mm_items,

@@ -1638,7 +1633,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1633 self,

1634 inputs: ProcessorInputs,

1635 timing_ctx: TimingContext,

1641 - ) -> MultiModalInputs:

1636 + ) -> MultiModalInput:

1637 """

1638 Process multi-modal inputs to be used in vLLM.

1639

@@ -1673,7 +1668,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1668 for modality, placeholders in mm_placeholders.items()

1669 }

1670

1676 - return mm_inputs(

1671 + return mm_input(

1672 prompt_token_ids=prompt_ids,

1673 mm_kwargs=mm_info.kwargs,

1674 mm_hashes=mm_info.hashes,

@@ -1708,7 +1703,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1703 self,

1704 prompt: str | list[int],

1705 mm_items: MultiModalDataItems,

1711 - encoder_inputs: MultiModalInputs,

1706 + encoder_inputs: MultiModalInput,

1707 ):

1708 tokenizer = self.info.get_tokenizer()

1709 decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)

@@ -1721,7 +1716,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1716 decoder_prompt_text = None

1717 decoder_prompt_ids = decoder_prompt_raw

1718

1724 - return mm_enc_dec_inputs(

1719 + return mm_enc_dec_input(

1720 encoder_inputs,

1721 decoder_prompt_ids,

1722 decoder_prompt=decoder_prompt_text,

@@ -1731,7 +1726,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1726 self,

1727 inputs: ProcessorInputs,

1728 timing_ctx: TimingContext,

1734 - ) -> MultiModalEncDecInputs:

1729 + ) -> MultiModalEncDecInput:

1730 """

1731 Process multi-modal inputs to be used in vLLM.

1732 The main processing steps are modified to fit encoder-decoder model:

vllm/multimodal/registry.py

+2 -2

@@ -7,6 +7,7 @@ from dataclasses import dataclass

7 from multiprocessing.synchronize import Lock as LockType

8 from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast

9

10 +from vllm.inputs import MultiModalInput

11 from vllm.logger import init_logger

12 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

13

@@ -19,7 +20,6 @@ from .cache import (

20 ShmObjectStoreReceiverCache,

21 ShmObjectStoreSenderCache,

22 )

22 -from .inputs import MultiModalInputs

23 from .processing import (

24 BaseDummyInputsBuilder,

25 BaseMultiModalProcessor,

@@ -220,7 +220,7 @@ class MultiModalRegistry:

220 *,

221 cache: BaseMultiModalProcessorCache | None = None,

222 processor: BaseMultiModalProcessor | None = None,

223 - ) -> MultiModalInputs:

223 + ) -> MultiModalInput:

224 """

225 Create dummy data for profiling the memory usage of a model.

226

vllm/multimodal/utils.py

+3 -3

@@ -12,6 +12,7 @@ import numpy.typing as npt

12 from PIL import Image

13 from typing_extensions import deprecated

14

15 +from vllm.inputs import MultiModalPlaceholders

16 from vllm.utils.import_utils import LazyLoader

17

18 from .hasher import MultiModalHasher

@@ -19,7 +20,6 @@ from .inputs import (

20 BatchedTensorInputs,

21 MultiModalFieldElem,

22 MultiModalKwargsItem,

22 - MultiModalPlaceholderDict,

23 MultiModalSharedField,

24 )

25 from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO

@@ -110,10 +110,10 @@ def encode_video_url(

110

111

112 def argsort_mm_positions(

113 - mm_positions: MultiModalPlaceholderDict,

113 + mm_positions: MultiModalPlaceholders,

114 ) -> list[tuple[str, int]]:

115 """

116 - Given a `MultiModalPlaceholderDict`, output a sequence of keys to

116 + Given a `MultiModalPlaceholders`, output a sequence of keys to

117 sort the dictionary by `offset` (starting index in the input sequence)

118 in ascending order.

119

vllm/platforms/interface.py

+2 -2

@@ -17,7 +17,7 @@ if TYPE_CHECKING:

17 from torch.distributed import PrefixStore, ProcessGroup

18

19 from vllm.config import VllmConfig

20 - from vllm.inputs import ProcessorInputs

20 + from vllm.inputs import EngineInput

21 from vllm.pooling_params import PoolingParams

22 from vllm.sampling_params import SamplingParams

23 from vllm.utils.argparse_utils import FlexibleArgumentParser

@@ -635,7 +635,7 @@ class Platform:

635 @classmethod

636 def validate_request(

637 cls,

638 - processed_inputs: "ProcessorInputs",

638 + processed_inputs: "EngineInput",

639 params: "SamplingParams | PoolingParams",

640 ) -> None:

641 """Raises if this request is unsupported on this platform"""

vllm/plugins/io_processors/interface.py

+1 -1

@@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Sequence

6 from typing import Generic, TypeVar

7

8 from vllm.config import VllmConfig

9 -from vllm.inputs.data import PromptType

9 +from vllm.inputs import PromptType

10 from vllm.outputs import PoolingRequestOutput

11 from vllm.pooling_params import PoolingParams

12 from vllm.renderers import BaseRenderer

vllm/renderers/base.py

+51 -58

@@ -11,17 +11,32 @@ from typing import TYPE_CHECKING, Any, Generic, overload

11 from typing_extensions import TypeVar

12

13 from vllm.inputs import (

14 - EmbedsInputs,

14 + EmbedsInput,

15 EmbedsPrompt,

16 - EncoderDecoderInputs,

17 - ProcessorInputs,

18 - SingletonInputs,

16 + EncoderDecoderInput,

17 + EngineInput,

18 + MultiModalDataDict,

19 + MultiModalInput,

20 + MultiModalUUIDDict,

21 + SingletonInput,

22 TextPrompt,

20 - TokenInputs,

23 + TokensInput,

24 TokensPrompt,

25 + build_enc_dec_input,

26 + embeds_input,

27 + tokens_input,

28 )

23 -from vllm.inputs.data import build_enc_dec_inputs, embeds_inputs, token_inputs

29 from vllm.logger import init_logger

30 +from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry

31 +from vllm.multimodal.cache import BaseMultiModalProcessorCache

32 +from vllm.multimodal.parse import (

33 + MultiModalDataItems,

34 + MultiModalUUIDItems,

35 + parse_mm_uuids,

36 +)

37 +from vllm.multimodal.processing import BaseMultiModalProcessor

38 +from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs

39 +from vllm.multimodal.registry import MultiModalTimingRegistry

40 from vllm.tokenizers import TokenizerLike

41 from vllm.utils.async_utils import AsyncMicrobatchTokenizer

42 from vllm.utils.counter import AtomicCounter

@@ -46,14 +61,6 @@ if TYPE_CHECKING:

61 ChatCompletionMessageParam,

62 ConversationMessage,

63 )

49 - from vllm.multimodal.cache import BaseMultiModalProcessorCache

50 - from vllm.multimodal.inputs import (

51 - MultiModalDataDict,

52 - MultiModalInputs,

53 - MultiModalUUIDDict,

54 - )

55 - from vllm.multimodal.parse import MultiModalDataItems, MultiModalUUIDItems

56 - from vllm.multimodal.processing import BaseMultiModalProcessor

64

65 logger = init_logger(__name__)

66

@@ -86,9 +93,6 @@ class BaseRenderer(ABC, Generic[_T]):

93 self.mm_processor: BaseMultiModalProcessor | None = None

94 self._mm_cache_stats: MultiModalCacheStats | None = None

95 if config.model_config.is_multimodal_model:

89 - from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry

90 - from vllm.multimodal.registry import MultiModalTimingRegistry

91 -

96 mm_processor_cache = mm_registry.processor_cache_from_config(config)

97

98 # Deep-copy the tokenizer so the multimodal processor gets its

@@ -524,9 +528,9 @@ class BaseRenderer(ABC, Generic[_T]):

528 # Step 4: Convert to engine inputs

529 def _validate_mm_uuids(

530 self,

527 - mm_data: "MultiModalDataDict",

528 - mm_data_items: "MultiModalDataItems",

529 - mm_uuid_items: "MultiModalUUIDItems",

531 + mm_data: MultiModalDataDict,

532 + mm_data_items: MultiModalDataItems,

533 + mm_uuid_items: MultiModalUUIDItems,

534 ) -> None:

535 # NOTE: Keys corresponding to `None` in `mm_data` don't appear in

536 # `mm_data_items`

@@ -560,11 +564,11 @@ class BaseRenderer(ABC, Generic[_T]):

564

565 def _process_mm_uuids(

566 self,

563 - mm_data: "MultiModalDataDict",

564 - mm_data_items: "MultiModalDataItems",

565 - mm_uuid_items: "MultiModalUUIDItems",

567 + mm_data: MultiModalDataDict,

568 + mm_data_items: MultiModalDataItems,

569 + mm_uuid_items: MultiModalUUIDItems,

570 mm_req_id: str,

567 - ):

571 + ) -> MultiModalUUIDItems:

572 model_config = self.model_config

573

574 # NOTE: When users explicitly turn off BOTH prefix caching and input

@@ -590,14 +594,11 @@ class BaseRenderer(ABC, Generic[_T]):

594 def _process_multimodal(

595 self,

596 prompt: list[int] | str,

593 - mm_data: "MultiModalDataDict",

594 - mm_uuids: "MultiModalUUIDDict | None",

597 + mm_data: MultiModalDataDict,

598 + mm_uuids: MultiModalUUIDDict | None,

599 mm_processor_kwargs: Mapping[str, object] | None,

600 tokenization_kwargs: dict[str, Any] | None,

597 - ) -> "MultiModalInputs":

598 - from vllm.multimodal.parse import parse_mm_uuids

599 - from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs

600 -

601 + ) -> "MultiModalInput":

602 mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"

603

604 mm_processor = self.get_mm_processor()

@@ -628,12 +629,12 @@ class BaseRenderer(ABC, Generic[_T]):

629 def _process_tokens(

630 self,

631 prompt: TokensPrompt,

631 - ) -> "TokenInputs | MultiModalInputs":

632 + ) -> TokensInput | MultiModalInput:

633 prompt_token_ids = prompt["prompt_token_ids"]

634

634 - inputs: TokenInputs | MultiModalInputs

635 + engine_input: TokensInput | MultiModalInput

636 if multi_modal_data := prompt.get("multi_modal_data"):

636 - inputs = self._process_multimodal(

637 + engine_input = self._process_multimodal(

638 prompt_token_ids,

639 multi_modal_data,

640 mm_processor_kwargs=prompt.get("mm_processor_kwargs"),

@@ -641,19 +642,16 @@ class BaseRenderer(ABC, Generic[_T]):

642 mm_uuids=prompt.get("multi_modal_uuids"),

643 )

644 else:

644 - inputs = token_inputs(prompt_token_ids)

645 + engine_input = tokens_input(prompt_token_ids)

646

647 if prompt_text := prompt.get("prompt"):

647 - inputs["prompt"] = prompt_text

648 + engine_input["prompt"] = prompt_text

649 if cache_salt := prompt.get("cache_salt"):

649 - inputs["cache_salt"] = cache_salt

650 + engine_input["cache_salt"] = cache_salt

651

651 - return inputs

652 + return engine_input

653

653 - def _process_embeds(

654 - self,

655 - prompt: EmbedsPrompt,

656 - ) -> EmbedsInputs:

654 + def _process_embeds(self, prompt: EmbedsPrompt) -> EmbedsInput:

655 if not self.model_config.enable_prompt_embeds:

656 raise ValueError(

657 "You must set `--enable-prompt-embeds` to input `prompt_embeds`."

@@ -676,15 +674,12 @@ class BaseRenderer(ABC, Generic[_T]):

674 # hidden device transfer in the critical path of generation.

675 prompt_embeds = prompt_embeds.cpu()

676

679 - return embeds_inputs(

677 + return embeds_input(

678 prompt_embeds=prompt_embeds,

679 cache_salt=prompt.get("cache_salt"),

680 )

681

684 - def _process_singleton(

685 - self,

686 - prompt: SingletonTokPrompt,

687 - ) -> SingletonInputs:

682 + def _process_singleton(self, prompt: SingletonTokPrompt) -> SingletonInput:

683 if "prompt_embeds" in prompt:

684 return self._process_embeds(prompt) # type: ignore[arg-type]

685

@@ -693,7 +688,7 @@ class BaseRenderer(ABC, Generic[_T]):

688 def _process_enc_dec(

689 self,

690 prompt: EncoderDecoderTokPrompt,

696 - ) -> EncoderDecoderInputs:

691 + ) -> EncoderDecoderInput:

692 enc_prompt = prompt["encoder_prompt"]

693 dec_prompt = prompt["decoder_prompt"]

694

@@ -704,27 +699,25 @@ class BaseRenderer(ABC, Generic[_T]):

699 if isinstance(self.mm_processor, EncDecMultiModalProcessor):

700 skip_decoder_start_token = self.mm_processor.skip_decoder_start_token

701

707 - return build_enc_dec_inputs(

708 - encoder_inputs=self._process_singleton(enc_prompt),

709 - decoder_inputs=(

702 + return build_enc_dec_input(

703 + encoder_input=self._process_singleton(enc_prompt),

704 + decoder_input=(

705 None if dec_prompt is None else self._process_singleton(dec_prompt)

706 ),

707 decoder_start_token_id=self.get_dec_start_token_id(),

708 skip_decoder_start_token=skip_decoder_start_token,

709 )

710

716 - def process_for_engine(

717 - self, prompt: TokPrompt, arrival_time: float

718 - ) -> ProcessorInputs:

719 - engine_prompt: ProcessorInputs

711 + def process_for_engine(self, prompt: TokPrompt, arrival_time: float) -> EngineInput:

712 + engine_input: EngineInput

713 if "encoder_prompt" in prompt:

721 - engine_prompt = self._process_enc_dec(prompt) # type: ignore[arg-type]

714 + engine_input = self._process_enc_dec(prompt) # type: ignore[arg-type]

715 else:

723 - engine_prompt = self._process_singleton(prompt)

716 + engine_input = self._process_singleton(prompt)

717

725 - engine_prompt["arrival_time"] = arrival_time

718 + engine_input["arrival_time"] = arrival_time

719

727 - return engine_prompt

720 + return engine_input

721

722 # Top-level methods

723 def render_cmpl(

vllm/renderers/hf.py

+7 -13

@@ -5,7 +5,7 @@ import itertools

5 from collections import defaultdict, deque

6 from collections.abc import Set

7 from functools import lru_cache

8 -from typing import TYPE_CHECKING, Any, Literal, cast, overload

8 +from typing import Any, Literal, cast, overload

9

10 import jinja2

11 import jinja2.ext

@@ -25,6 +25,7 @@ from vllm.entrypoints.chat_utils import (

25 parse_chat_messages,

26 parse_chat_messages_async,

27 )

28 +from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict

29 from vllm.logger import init_logger

30 from vllm.tokenizers import cached_get_tokenizer

31 from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer

@@ -37,13 +38,6 @@ from .inputs import DictPrompt

38 from .inputs.preprocess import parse_dec_only_prompt

39 from .params import ChatParams

40

40 -if TYPE_CHECKING:

41 - from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict

42 -else:

43 - MultiModalDataDict = dict[str, Any]

44 - MultiModalUUIDDict = dict[str, Any]

45 -

46 -

41 logger = init_logger(__name__)

42

43

@@ -512,9 +506,9 @@ def safe_apply_chat_template(

506

507

508 def rebuild_mm_uuids_from_mm_data(

515 - mm_uuids: "MultiModalUUIDDict",

516 - mm_data: "MultiModalDataDict",

517 -) -> "MultiModalUUIDDict":

509 + mm_uuids: MultiModalUUIDDict,

510 + mm_data: MultiModalDataDict,

511 +) -> MultiModalUUIDDict:

512 """Rebuild mm_uuids after vision_chunk processing.

513

514 When videos are split into chunks, the original UUIDs need to be updated

@@ -547,7 +541,7 @@ def rebuild_mm_uuids_from_mm_data(

541

542

543 def build_video_prompts_from_mm_data(

550 - mm_data: "MultiModalDataDict",

544 + mm_data: MultiModalDataDict,

545 ) -> list[str]:

546 """Build video prompts from vision_chunk data.

547

@@ -585,7 +579,7 @@ def build_video_prompts_from_mm_data(

579

580 def replace_vision_chunk_video_placeholder(

581 prompt_raw: str | list[int],

588 - mm_data: "MultiModalDataDict",

582 + mm_data: MultiModalDataDict,

583 video_placeholder: str | None,

584 ) -> str | list[int]:

585 # get video placeholder, replace it with runtime video-chunk prompts

vllm/renderers/inputs/preprocess.py

+10 -9

@@ -9,8 +9,8 @@ from typing import TYPE_CHECKING, NamedTuple, TypeAlias, TypedDict, overload

9

10 from vllm.inputs import (

11 EmbedsPrompt,

12 + EngineInput,

13 ExplicitEncoderDecoderPrompt,

13 - ProcessorInputs,

14 PromptType,

15 SingletonPrompt,

16 TextPrompt,

@@ -70,28 +70,28 @@ def conversation_to_seq(

70

71 DecoderOnlyDictPrompt: TypeAlias = TextPrompt | TokensPrompt | EmbedsPrompt

72 """

73 -A [`DecoderOnlyPrompt`][vllm.inputs.data.DecoderOnlyPrompt]

73 +A [`DecoderOnlyPrompt`][vllm.inputs.llm.DecoderOnlyPrompt]

74 that has been standardized into a dictionary.

75 """

76

77

78 EncoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt

79 """

80 -A [`EncoderPrompt`][vllm.inputs.data.EncoderPrompt]

80 +A [`EncoderPrompt`][vllm.inputs.llm.EncoderPrompt]

81 that has been standardized into a dictionary.

82 """

83

84

85 DecoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt

86 """

87 -A [`DecoderPrompt`][vllm.inputs.data.DecoderPrompt]

87 +A [`DecoderPrompt`][vllm.inputs.llm.DecoderPrompt]

88 that has been standardized into a dictionary.

89 """

90

91

92 class EncoderDecoderDictPrompt(TypedDict):

93 """

94 - A [`EncoderDecoderPrompt`][vllm.inputs.data.EncoderDecoderPrompt]

94 + A [`EncoderDecoderPrompt`][vllm.inputs.llm.EncoderDecoderPrompt]

95 that has been standardized into a dictionary.

96 """

97

@@ -104,14 +104,14 @@ SingletonDictPrompt: TypeAlias = (

104 DecoderOnlyDictPrompt | EncoderDictPrompt | DecoderDictPrompt

105 )

106 """

107 -A [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt]

107 +A [`SingletonPrompt`][vllm.inputs.llm.SingletonPrompt]

108 that has been standardized into a dictionary.

109 """

110

111

112 DictPrompt: TypeAlias = DecoderOnlyDictPrompt | EncoderDecoderDictPrompt

113 """

114 -A [`PromptType`][vllm.inputs.data.PromptType]

114 +A [`PromptType`][vllm.inputs.llm.PromptType]

115 that has been standardized into a dictionary.

116 """

117

@@ -236,7 +236,7 @@ def extract_target_prompt(model_config: "ModelConfig", prompt: object):

236

237 def extract_prompt_components(

238 model_config: "ModelConfig",

239 - prompt: PromptType | ProcessorInputs,

239 + prompt: PromptType | EngineInput,

240 ) -> PromptComponents:

241 target_prompt = extract_target_prompt(model_config, prompt)

242

@@ -248,7 +248,8 @@ def extract_prompt_components(

248

249

250 def extract_prompt_len(

251 - model_config: "ModelConfig", prompt: PromptType | ProcessorInputs

251 + model_config: "ModelConfig",

252 + prompt: PromptType | EngineInput,

253 ):

254 target_prompt = extract_target_prompt(model_config, prompt)

255

vllm/v1/engine/async_llm.py

+5 -5

@@ -21,7 +21,7 @@ from vllm.distributed.weight_transfer.base import (

21 from vllm.engine.arg_utils import AsyncEngineArgs

22 from vllm.engine.protocol import EngineClient, StreamingInput

23 from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep

24 -from vllm.inputs import ProcessorInputs, PromptType

24 +from vllm.inputs import EngineInput, PromptType

25 from vllm.logger import init_logger

26 from vllm.lora.request import LoRARequest

27 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

@@ -139,7 +139,7 @@ class AsyncLLM(EngineClient):

139 self.model_config.io_processor_plugin,

140 )

141

142 - # Convert TokPrompt --> EngineCoreRequest.

142 + # Convert EngineInput --> EngineCoreRequest.

143 self.input_processor = InputProcessor(self.vllm_config, renderer)

144

145 # Converts EngineCoreOutputs --> RequestOutput.

@@ -290,7 +290,7 @@ class AsyncLLM(EngineClient):

290 request_id: str,

291 prompt: EngineCoreRequest

292 | PromptType

293 - | ProcessorInputs

293 + | EngineInput

294 | AsyncGenerator[StreamingInput, None],

295 params: SamplingParams | PoolingParams,

296 arrival_time: float | None = None,

@@ -530,7 +530,7 @@ class AsyncLLM(EngineClient):

530 self,

531 prompt: EngineCoreRequest

532 | PromptType

533 - | ProcessorInputs

533 + | EngineInput

534 | AsyncGenerator[StreamingInput, None],

535 sampling_params: SamplingParams,

536 request_id: str,

@@ -776,7 +776,7 @@ class AsyncLLM(EngineClient):

776

777 async def encode(

778 self,

779 - prompt: PromptType | ProcessorInputs,

779 + prompt: PromptType | EngineInput,

780 pooling_params: PoolingParams,

781 request_id: str,

782 lora_request: LoRARequest | None = None,

vllm/v1/engine/input_processor.py

+19 -23

@@ -7,20 +7,18 @@ from typing import Any, Literal

7

8 import vllm.envs as envs

9 from vllm.config import VllmConfig

10 -from vllm.inputs.data import (

11 - ProcessorInputs,

10 +from vllm.inputs import (

11 + EngineInput,

12 PromptType,

13 - SingletonInputs,

13 + SingletonInput,

14 + split_enc_dec_input,

15 )

15 -from vllm.inputs.parse import split_enc_dec_inputs

16 from vllm.inputs.preprocess import InputPreprocessor

17 from vllm.logger import init_logger

18 from vllm.lora.request import LoRARequest

19 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

20 from vllm.multimodal.encoder_budget import MultiModalBudget

21 -from vllm.multimodal.inputs import (

22 - MultiModalFeatureSpec,

23 -)

21 +from vllm.multimodal.inputs import MultiModalFeatureSpec

22 from vllm.multimodal.utils import argsort_mm_positions

23 from vllm.platforms import current_platform

24 from vllm.pooling_params import PoolingParams

@@ -197,7 +195,7 @@ class InputProcessor:

195 def process_inputs(

196 self,

197 request_id: str,

200 - prompt: PromptType | ProcessorInputs,

198 + prompt: PromptType | EngineInput,

199 params: SamplingParams | PoolingParams,

200 supported_tasks: tuple[SupportedTask, ...],

201 arrival_time: float | None = None,

@@ -232,7 +230,7 @@ class InputProcessor:

230 if arrival_time is None:

231 arrival_time = prompt.get("arrival_time", time.time()) # type: ignore[assignment]

232

235 - processed_inputs: ProcessorInputs = prompt # type: ignore[assignment]

233 + processed_inputs: EngineInput = prompt # type: ignore[assignment]

234 else:

235 logger.warning_once(

236 "Passing raw prompts to InputProcessor is deprecated "

@@ -250,7 +248,7 @@ class InputProcessor:

248

249 current_platform.validate_request(processed_inputs, params)

250

253 - encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)

251 + encoder_inputs, decoder_inputs = split_enc_dec_input(processed_inputs)

252 self._validate_model_inputs(encoder_inputs, decoder_inputs)

253

254 # Mypy can be conservative for TypedDict unions; normalize access.

@@ -385,7 +383,7 @@ class InputProcessor:

383

384 def _validate_model_input(

385 self,

388 - prompt_inputs: SingletonInputs,

386 + prompt_input: SingletonInput,

387 prompt_type: Literal["encoder", "decoder"],

388 ) -> None:

389 model_config = self.model_config

@@ -393,20 +391,18 @@ class InputProcessor:

391

392 prompt_ids = (

393 None

396 - if prompt_inputs["type"] == "embeds"

397 - else prompt_inputs["prompt_token_ids"]

394 + if prompt_input["type"] == "embeds"

395 + else prompt_input["prompt_token_ids"]

396 )

397 prompt_embeds = (

400 - prompt_inputs["prompt_embeds"]

401 - if prompt_inputs["type"] == "embeds"

402 - else None

398 + prompt_input["prompt_embeds"] if prompt_input["type"] == "embeds" else None

399 )

400

401 prompt_len = length_from_prompt_token_ids_or_embeds(prompt_ids, prompt_embeds)

402 self._validate_prompt_len(prompt_len, prompt_type)

403

408 - if prompt_inputs["type"] == "multimodal":

409 - decoder_mm_positions = prompt_inputs["mm_placeholders"]

404 + if prompt_input["type"] == "multimodal":

405 + decoder_mm_positions = prompt_input["mm_placeholders"]

406 for modality, mm_positions in decoder_mm_positions.items():

407 for mm_position in mm_positions:

408 embed_length = mm_position.get_num_embeds()

@@ -439,10 +435,10 @@ class InputProcessor:

435

436 def _validate_model_inputs(

437 self,

442 - encoder_inputs: SingletonInputs | None,

443 - decoder_inputs: SingletonInputs,

438 + encoder_input: SingletonInput | None,

439 + decoder_input: SingletonInput,

440 ):

445 - if encoder_inputs is not None:

446 - self._validate_model_input(encoder_inputs, prompt_type="encoder")

441 + if encoder_input is not None:

442 + self._validate_model_input(encoder_input, prompt_type="encoder")

443

448 - self._validate_model_input(decoder_inputs, prompt_type="decoder")

444 + self._validate_model_input(decoder_input, prompt_type="decoder")

vllm/v1/engine/llm_engine.py

+3 -3

@@ -14,7 +14,7 @@ from vllm.config import ParallelConfig, VllmConfig

14 from vllm.distributed import stateless_destroy_torch_distributed_process_group

15 from vllm.distributed.parallel_state import get_dp_group

16 from vllm.engine.arg_utils import EngineArgs

17 -from vllm.inputs import ProcessorInputs, PromptType

17 +from vllm.inputs import EngineInput, PromptType

18 from vllm.logger import init_logger

19 from vllm.lora.request import LoRARequest

20 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

@@ -96,7 +96,7 @@ class LLMEngine:

96 self.model_config.io_processor_plugin,

97 )

98

99 - # Convert TokPrompt --> EngineCoreRequest.

99 + # Convert EngineInput --> EngineCoreRequest.

100 self.input_processor = InputProcessor(self.vllm_config, renderer)

101

102 # Converts EngineCoreOutputs --> RequestOutput.

@@ -216,7 +216,7 @@ class LLMEngine:

216 def add_request(

217 self,

218 request_id: str,

219 - prompt: EngineCoreRequest | PromptType | ProcessorInputs,

219 + prompt: EngineCoreRequest | PromptType | EngineInput,

220 params: SamplingParams | PoolingParams,

221 arrival_time: float | None = None,

222 lora_request: LoRARequest | None = None,

[Misc] Reorganize inputs (#35182)