@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cee711fdbb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core] Rename input data types (#8688)

Cyrus Leung committed Oct 16, 2024 at 18:49 UTC cee711fdbb88d7c6506a9039d74cb2911c516f94
32 files changed +438 -340
docs/source/dev/input_processing/model_inputs_index.rst
+1 -1
@@ -25,7 +25,7 @@ Module Contents
25 LLM Engine Inputs
26 -----------------
27
28 -.. autoclass:: vllm.inputs.LLMInputs
28 +.. autoclass:: vllm.inputs.DecoderOnlyInputs
29 :members:
30 :show-inheritance:
31
tests/models/decoder_only/vision_language/test_phi3v.py
+13 -15
@@ -1,12 +1,12 @@
1 import os
2 import re
3 -from typing import Callable, List, Optional, Tuple, Type
3 +from typing import List, Optional, Tuple, Type
4
5 import pytest
6 import torch
7 from transformers import AutoImageProcessor, AutoTokenizer
8
9 -from vllm.inputs import InputContext, LLMInputs
9 +from vllm.inputs import InputContext, token_inputs
10 from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
11 from vllm.multimodal import MultiModalRegistry
12 from vllm.multimodal.utils import rescale_image_size
@@ -311,7 +311,7 @@ def test_input_mapper_override(model: str, image_assets: _ImageAssets,
311 (4, 781),
312 (16, 2653),
313 ])
314 -def test_max_tokens_override(get_max_phi3v_image_tokens: Callable, model: str,
314 +def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
315 num_crops: int, expected_max_tokens: int):
316 """Ensure get_max_phi3v_image_tokens handles num_crops properly."""
317 # NOTE: mm_processor_kwargs on the context in this test is unused, since
@@ -343,8 +343,8 @@ def test_max_tokens_override(get_max_phi3v_image_tokens: Callable, model: str,
343 (16, 2653, 1),
344 (16, 2653, 2),
345 ])
346 -def test_dummy_data_override(dummy_data_for_phi3v: Callable, model: str,
347 - num_crops: int, toks_per_img: int, num_imgs: int):
346 +def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
347 + toks_per_img: int, num_imgs: int):
348 """Ensure dummy_data_for_phi3v handles num_crops properly."""
349 # Same as the previous test - don't initialize mm_processor_kwargs
350 # in this test and assume that the kwargs will be correctly expanded by
@@ -374,7 +374,7 @@ def test_dummy_data_override(dummy_data_for_phi3v: Callable, model: str,
374 (16, 1921, 1),
375 (16, 1921, 2),
376 ])
377 -def test_input_processor_override(input_processor_for_phi3v: Callable,
377 +def test_input_processor_override(input_processor_for_phi3v,
378 image_assets: _ImageAssets, model: str,
379 num_crops: int, expected_toks_per_img: int,
380 num_imgs: int):
@@ -393,16 +393,14 @@ def test_input_processor_override(input_processor_for_phi3v: Callable,
393 prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
394 images = [image_assets[0].pil_image] * num_imgs
395
396 - llm_inputs = LLMInputs(prompt_token_ids=tokenizer.encode(prompt),
397 - prompt=prompt,
398 - multi_modal_data={"image": images})
396 + inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
397 + prompt=prompt,
398 + multi_modal_data={"image": images})
399
400 - proc_llm_inputs = input_processor_for_phi3v(
401 - ctx=ctx,
402 - llm_inputs=llm_inputs,
403 - num_crops=num_crops,
404 - )
400 + processed_inputs = input_processor_for_phi3v(ctx,
401 + inputs,
402 + num_crops=num_crops)
403
404 # Ensure we have the right number of placeholders per num_crops size
407 - img_tok_count = proc_llm_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
405 + img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
406 assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/decoder_only/vision_language/test_qwen.py
+6 -6
@@ -5,7 +5,7 @@ import pytest
5 import torch
6 from PIL.Image import Image
7
8 -from vllm.inputs import InputContext, LLMInputs
8 +from vllm.inputs import InputContext, token_inputs
9 from vllm.multimodal.base import MultiModalInputs
10 from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
11
@@ -71,12 +71,12 @@ def test_input_processor_valid_mm_data(input_processor_for_qwen,
71 """Happy cases for image inputs to Qwen's multimodal input processor."""
72 prompt = "".join(
73 [f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
74 - inputs = LLMInputs(
74 + inputs = token_inputs(
75 prompt=prompt,
76 # When processing multimodal data for a multimodal model, the qwen
77 # input processor will overwrite the provided prompt_token_ids with
78 # the image prompts
79 - prompt_token_ids=None,
79 + prompt_token_ids=[],
80 multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
81 )
82 proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
@@ -134,9 +134,9 @@ def test_input_processor_invalid_mm_data(input_processor_for_qwen,
134 trust_remote_code=True)
135 prompt = "Picture 1: <img></img>\n"
136 prompt_token_ids = tokenizer.encode(prompt)
137 - inputs = LLMInputs(prompt=prompt,
138 - prompt_token_ids=prompt_token_ids,
139 - multi_modal_data=mm_data)
137 + inputs = token_inputs(prompt=prompt,
138 + prompt_token_ids=prompt_token_ids,
139 + multi_modal_data=mm_data)
140 # Should fail since we have too many or too few dimensions for embeddings
141 with pytest.raises(ValueError):
142 input_processor_for_qwen(qwen_vl_context, inputs)
tests/multimodal/test_processor_kwargs.py
+9 -9
@@ -5,7 +5,7 @@ from unittest.mock import patch
5 import pytest
6 import torch
7
8 -from vllm.inputs import InputContext, LLMInputs
8 +from vllm.inputs import DecoderOnlyInputs, InputContext, token_inputs
9 from vllm.inputs.registry import InputRegistry
10 from vllm.multimodal import MultiModalRegistry
11 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
@@ -31,7 +31,7 @@ def use_processor_mock():
31 """Patches the internal model input processor with an override callable."""
32
33 def custom_processor(ctx: InputContext,
34 - llm_inputs: LLMInputs,
34 + inputs: DecoderOnlyInputs,
35 *,
36 num_crops=DEFAULT_NUM_CROPS):
37 # For testing purposes, we don't worry about the llm inputs / return
@@ -84,7 +84,7 @@ def test_default_processor_is_a_noop():
84 dummy_registry = InputRegistry()
85 ctx = build_model_context(DUMMY_MODEL_ID)
86 processor = dummy_registry.create_input_processor(ctx.model_config)
87 - proc_inputs = LLMInputs(prompt_token_ids=[], prompt="")
87 + proc_inputs = token_inputs(prompt_token_ids=[], prompt="")
88 proc_outputs = processor(inputs=proc_inputs)
89 assert proc_inputs is proc_outputs
90
@@ -125,9 +125,9 @@ def test_input_processor_kwargs(use_processor_mock, init_num_crops,
125 ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
126 processor = dummy_registry.create_input_processor(ctx.model_config)
127 num_crops_val = processor(
128 - LLMInputs(prompt_token_ids=[],
129 - prompt="",
130 - mm_processor_kwargs=inference_kwargs))
128 + token_inputs(prompt_token_ids=[],
129 + prompt="",
130 + mm_processor_kwargs=inference_kwargs))
131 assert num_crops_val == expected_seq_count
132
133
@@ -154,9 +154,9 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
154 processor = dummy_registry.create_input_processor(ctx.model_config)
155 # Should filter out the inference time kwargs
156 num_crops_val = processor(
157 - LLMInputs(prompt_token_ids=[],
158 - prompt="",
159 - mm_processor_kwargs=mm_processor_kwargs))
157 + token_inputs(prompt_token_ids=[],
158 + prompt="",
159 + mm_processor_kwargs=mm_processor_kwargs))
160 assert num_crops_val == DEFAULT_NUM_CROPS
161
162
vllm/engine/llm_engine.py
+5 -5
@@ -29,8 +29,8 @@ from vllm.entrypoints.openai.logits_processors import get_logits_processors
29 from vllm.executor.executor_base import ExecutorBase
30 from vllm.executor.gpu_executor import GPUExecutor
31 from vllm.executor.ray_utils import initialize_ray_cluster
32 -from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
33 - InputRegistry, LLMInputs, PromptType)
32 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
33 + EncoderDecoderInputs, InputRegistry, PromptType)
34 from vllm.inputs.preprocess import InputPreprocessor
35 from vllm.logger import init_logger
36 from vllm.lora.request import LoRARequest
@@ -635,7 +635,7 @@ class LLMEngine:
635 def _add_processed_request(
636 self,
637 request_id: str,
638 - processed_inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
638 + processed_inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
639 params: Union[SamplingParams, PoolingParams],
640 arrival_time: float,
641 lora_request: Optional[LoRARequest],
@@ -1855,8 +1855,8 @@ class LLMEngine:
1855 def is_embedding_model(self):
1856 return self.model_config.is_embedding_model
1857
1858 - def _validate_model_inputs(self, inputs: Union[LLMInputs,
1859 - EncoderDecoderLLMInputs]):
1858 + def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1859 + EncoderDecoderInputs]):
1860 if self.model_config.is_multimodal_model:
1861 # For encoder-decoder multimodal models, the max_prompt_len
1862 # restricts the decoder prompt length
vllm/inputs/__init__.py
+29 -8
@@ -1,7 +1,8 @@
1 -from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2 - LLMInputs, PromptType, SingletonPrompt, TextPrompt,
3 - TokensPrompt, build_explicit_enc_dec_prompt,
4 - to_enc_dec_tuple_list, zip_enc_dec_prompts)
1 +from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
2 + ExplicitEncoderDecoderPrompt, PromptType, SingletonInputs,
3 + SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
4 + build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
5 + token_inputs, zip_enc_dec_prompts)
6 from .registry import InputContext, InputRegistry
7
8 INPUT_REGISTRY = InputRegistry()
@@ -19,8 +20,11 @@ __all__ = [
20 "PromptType",
21 "SingletonPrompt",
22 "ExplicitEncoderDecoderPrompt",
22 - "LLMInputs",
23 - "EncoderDecoderLLMInputs",
23 + "TokenInputs",
24 + "token_inputs",
25 + "SingletonInputs",
26 + "DecoderOnlyInputs",
27 + "EncoderDecoderInputs",
28 "build_explicit_enc_dec_prompt",
29 "to_enc_dec_tuple_list",
30 "zip_enc_dec_prompts",
@@ -31,9 +35,9 @@ __all__ = [
35
36
37 def __getattr__(name: str):
34 - if name == "PromptInput":
35 - import warnings
38 + import warnings
39
40 + if name == "PromptInput":
41 msg = ("PromptInput has been renamed to PromptType. "
42 "The original name will be removed in an upcoming version.")
43
@@ -41,4 +45,21 @@ def __getattr__(name: str):
45
46 return PromptType
47
48 + if name == "LLMInputs":
49 + msg = ("LLMInputs has been renamed to DecoderOnlyInputs. "
50 + "The original name will be removed in an upcoming version.")
51 +
52 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
53 +
54 + return DecoderOnlyInputs
55 +
56 + if name == "EncoderDecoderLLMInputs":
57 + msg = (
58 + "EncoderDecoderLLMInputs has been renamed to EncoderDecoderInputs. "
59 + "The original name will be removed in an upcoming version.")
60 +
61 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
62 +
63 + return EncoderDecoderInputs
64 +
65 raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/data.py
+62 -16
@@ -1,5 +1,5 @@
1 from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
2 - Optional, Tuple, Union)
2 + Optional, Tuple, Union, cast)
3
4 from typing_extensions import NotRequired, TypedDict, TypeVar
5
@@ -51,7 +51,7 @@ class TokensPrompt(TypedDict):
51
52 SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
53 """
54 -Set of possible schemas for a single LLM input:
54 +Set of possible schemas for a single prompt:
55
56 - A text prompt (:class:`str` or :class:`TextPrompt`)
57 - A tokenized prompt (:class:`TokensPrompt`)
@@ -120,13 +120,8 @@ both decoder-only and encoder/decoder input types:
120 """
121
122
123 -class LLMInputs(TypedDict):
124 - """
125 - The inputs in :class:`~vllm.LLMEngine` before they are
126 - passed to the model executor.
127 -
128 - This specifies the data required for decoder-only models.
129 - """
123 +class TokenInputs(TypedDict):
124 + """Represents token-based inputs."""
125 prompt_token_ids: List[int]
126 """The token IDs of the prompt."""
127
@@ -150,7 +145,40 @@ class LLMInputs(TypedDict):
145 """
146
147
153 -class EncoderDecoderLLMInputs(LLMInputs):
148 +def token_inputs(
149 + prompt_token_ids: List[int],
150 + prompt: Optional[str] = None,
151 + multi_modal_data: Optional["MultiModalDataDict"] = None,
152 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
153 +) -> TokenInputs:
154 + """Construct :class:`TokenInputs` from optional values."""
155 + inputs = TokenInputs(prompt_token_ids=prompt_token_ids)
156 +
157 + if prompt is not None:
158 + inputs["prompt"] = prompt
159 + if multi_modal_data is not None:
160 + inputs["multi_modal_data"] = multi_modal_data
161 + if mm_processor_kwargs is not None:
162 + inputs["mm_processor_kwargs"] = mm_processor_kwargs
163 +
164 + return inputs
165 +
166 +
167 +SingletonInputs = TokenInputs
168 +"""
169 +A processed :class:`SingletonPrompt` which can be passed to
170 +:class:`vllm.sequence.Sequence`.
171 +"""
172 +
173 +DecoderOnlyInputs = TokenInputs
174 +"""
175 +The inputs in :class:`~vllm.LLMEngine` before they are
176 +passed to the model executor.
177 +This specifies the data required for decoder-only models.
178 +"""
179 +
180 +
181 +class EncoderDecoderInputs(TokenInputs):
182 """
183 The inputs in :class:`~vllm.LLMEngine` before they are
184 passed to the model executor.
@@ -204,11 +232,12 @@ def zip_enc_dec_prompts(
232 be zipped with the encoder/decoder prompts.
233 """
234 if mm_processor_kwargs is None:
207 - mm_processor_kwargs = {}
208 - if isinstance(mm_processor_kwargs, Dict):
235 + mm_processor_kwargs = cast(Dict[str, Any], {})
236 + if isinstance(mm_processor_kwargs, dict):
237 return [
210 - build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
211 - mm_processor_kwargs)
238 + build_explicit_enc_dec_prompt(
239 + encoder_prompt, decoder_prompt,
240 + cast(Dict[str, Any], mm_processor_kwargs))
241 for (encoder_prompt,
242 decoder_prompt) in zip(enc_prompts, dec_prompts)
243 ]
@@ -229,9 +258,9 @@ def to_enc_dec_tuple_list(
258
259
260 def __getattr__(name: str):
232 - if name == "PromptInput":
233 - import warnings
261 + import warnings
262
263 + if name == "PromptInput":
264 msg = ("PromptInput has been renamed to PromptType. "
265 "The original name will be removed in an upcoming version.")
266
@@ -239,4 +268,21 @@ def __getattr__(name: str):
268
269 return PromptType
270
271 + if name == "LLMInputs":
272 + msg = ("LLMInputs has been renamed to DecoderOnlyInputs. "
273 + "The original name will be removed in an upcoming version.")
274 +
275 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
276 +
277 + return DecoderOnlyInputs
278 +
279 + if name == "EncoderDecoderLLMInputs":
280 + msg = (
281 + "EncoderDecoderLLMInputs has been renamed to EncoderDecoderInputs. "
282 + "The original name will be removed in an upcoming version.")
283 +
284 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
285 +
286 + return EncoderDecoderInputs
287 +
288 raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/parse.py
+6 -6
@@ -4,9 +4,9 @@ from typing_extensions import TypeIs
4
5 from vllm.utils import is_list_of
6
7 -from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8 - LLMInputs, PromptType, SingletonPrompt, TextPrompt,
9 - TokensPrompt)
7 +from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
8 + ExplicitEncoderDecoderPrompt, PromptType, SingletonPrompt,
9 + TextPrompt, TokensPrompt)
10
11
12 class ParsedText(TypedDict):
@@ -100,7 +100,7 @@ def is_explicit_encoder_decoder_prompt(
100 return isinstance(prompt, dict) and "encoder_prompt" in prompt
101
102
103 -def is_valid_encoder_decoder_llm_inputs(
104 - inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
105 -) -> TypeIs[EncoderDecoderLLMInputs]:
103 +def is_encoder_decoder_inputs(
104 + inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
105 +) -> TypeIs[EncoderDecoderInputs]:
106 return "encoder_prompt_token_ids" in inputs
vllm/inputs/preprocess.py
+18 -18
@@ -10,7 +10,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11 from vllm.utils import print_warning_once
12
13 -from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
13 +from .data import (DecoderOnlyInputs, EncoderDecoderInputs, PromptType,
14 SingletonPrompt)
15 from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
16
@@ -306,7 +306,7 @@ class InputPreprocessor:
306 encoder_comps: PromptComponents,
307 decoder_comps: DecoderPromptComponents,
308 mm_processor_kwargs: Dict[str, Any],
309 - ) -> EncoderDecoderLLMInputs:
309 + ) -> EncoderDecoderInputs:
310 encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311 decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312
@@ -324,7 +324,7 @@ class InputPreprocessor:
324 decoder_prompt_ids,
325 force_bos=(encoder_mm_data is None and decoder_mm_data is None)))
326
327 - return EncoderDecoderLLMInputs(
327 + return EncoderDecoderInputs(
328 prompt_token_ids=decoder_prompt_ids,
329 prompt=decoder_prompt,
330 multi_modal_data=decoder_mm_data,
@@ -338,11 +338,11 @@ class InputPreprocessor:
338 self,
339 prompt: PromptType,
340 request_id: str,
341 - ) -> EncoderDecoderLLMInputs:
341 + ) -> EncoderDecoderInputs:
342 '''
343 For encoder/decoder models only:
344 Process an input prompt into an
345 - :class:`EncoderDecoderLLMInputs` instance.
345 + :class:`EncoderDecoderInputs` instance.
346
347 There are two types of input prompts:
348 singleton prompts which carry only the
@@ -369,7 +369,7 @@ class InputPreprocessor:
369
370 Returns:
371
372 - * :class:`EncoderDecoderLLMInputs` instance
372 + * :class:`EncoderDecoderInputs` instance
373 '''
374
375 encoder_comps: PromptComponents
@@ -411,7 +411,7 @@ class InputPreprocessor:
411 self,
412 prompt: PromptType,
413 request_id: str,
414 - ) -> EncoderDecoderLLMInputs:
414 + ) -> EncoderDecoderInputs:
415 """Async version of :meth:`_process_encoder_decoder_prompt`."""
416 encoder_comps: PromptComponents
417 decoder_comps: DecoderPromptComponents
@@ -455,17 +455,17 @@ class InputPreprocessor:
455 self,
456 prompt_comps: PromptComponents,
457 prompt_adapter_request: Optional[PromptAdapterRequest],
458 - ) -> LLMInputs:
458 + ) -> DecoderOnlyInputs:
459 (prompt, prompt_token_ids, multi_modal_data,
460 mm_processor_kwargs) = prompt_comps
461
462 prompt_token_ids = self._apply_prompt_adapter(
463 prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
464
465 - return LLMInputs(prompt_token_ids=prompt_token_ids,
466 - prompt=prompt,
467 - multi_modal_data=multi_modal_data,
468 - mm_processor_kwargs=mm_processor_kwargs)
465 + return DecoderOnlyInputs(prompt_token_ids=prompt_token_ids,
466 + prompt=prompt,
467 + multi_modal_data=multi_modal_data,
468 + mm_processor_kwargs=mm_processor_kwargs)
469
470 def _process_decoder_only_prompt(
471 self,
@@ -473,10 +473,10 @@ class InputPreprocessor:
473 request_id: str,
474 lora_request: Optional[LoRARequest] = None,
475 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
476 - ) -> LLMInputs:
476 + ) -> DecoderOnlyInputs:
477 '''
478 For decoder-only models:
479 - Process an input prompt into an :class:`LLMInputs` instance.
479 + Process an input prompt into an :class:`DecoderOnlyInputs` instance.
480
481 Arguments:
482
@@ -487,7 +487,7 @@ class InputPreprocessor:
487
488 Returns:
489
490 - * :class:`LLMInputs` instance
490 + * :class:`DecoderOnlyInputs` instance
491 '''
492
493 prompt_comps = self._extract_prompt_components(
@@ -507,7 +507,7 @@ class InputPreprocessor:
507 request_id: str,
508 lora_request: Optional[LoRARequest] = None,
509 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
510 - ) -> LLMInputs:
510 + ) -> DecoderOnlyInputs:
511 """Async version of :meth:`_process_decoder_only_prompt`."""
512 prompt_comps = await self._extract_prompt_components_async(
513 prompt,
@@ -526,7 +526,7 @@ class InputPreprocessor:
526 request_id: str,
527 lora_request: Optional[LoRARequest] = None,
528 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
529 - ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
529 + ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
530 """Preprocess the input prompt."""
531 if self.is_encoder_decoder_model():
532 # Encoder-decoder model requires special mapping of
@@ -554,7 +554,7 @@ class InputPreprocessor:
554 request_id: str,
555 lora_request: Optional[LoRARequest] = None,
556 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
557 - ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
557 + ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
558 """Async version of :meth:`preprocess`."""
559 if self.is_encoder_decoder_model():
560 # Encoder-decoder model requires special mapping of
vllm/inputs/registry.py
+9 -6
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
12 from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
13 resolve_mm_processor_kwargs)
14
15 -from .data import LLMInputs
15 +from .data import DecoderOnlyInputs
16
17 if TYPE_CHECKING:
18 from vllm.config import ModelConfig
@@ -100,7 +100,7 @@ class _MultiModalCounts(UserDict):
100 raise KeyError(msg) from exc
101
102
103 -InputProcessor = Callable[[InputContext, LLMInputs], LLMInputs]
103 +InputProcessor = Callable[[InputContext, DecoderOnlyInputs], DecoderOnlyInputs]
104 """Preprocess the inputs to the model."""
105
106
@@ -134,7 +134,7 @@ class InputRegistry:
134 # Avoid circular import
135 from vllm.sequence import SequenceData
136
137 - dummy_seq_data = SequenceData.from_token_counts((0, seq_len))
137 + dummy_seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
138 dummy_multi_modal_data = None
139
140 return dummy_seq_data, dummy_multi_modal_data
@@ -245,8 +245,11 @@ class InputRegistry:
245
246 return seq_data, mm_data
247
248 - def _default_input_processor(self, ctx: InputContext,
249 - inputs: LLMInputs) -> LLMInputs:
248 + def _default_input_processor(
249 + self,
250 + ctx: InputContext,
251 + inputs: DecoderOnlyInputs,
252 + ) -> DecoderOnlyInputs:
253 """The default input processor is a no-op."""
254 return inputs
255
@@ -279,7 +282,7 @@ class InputRegistry:
282 .get(model_cls, self._default_input_processor)
283
284 def process_input(self, model_config: "ModelConfig",
282 - inputs: LLMInputs) -> LLMInputs:
285 + inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
286 """
287 Apply an input processor to an instance of model inputs.
288
vllm/model_executor/models/blip.py
+10 -10
@@ -10,7 +10,7 @@ from transformers.models.blip.modeling_blip import BlipAttention
10
11 from vllm.config import ModelConfig
12 from vllm.distributed import divide, get_tensor_model_parallel_world_size
13 -from vllm.inputs import LLMInputs
13 +from vllm.inputs import DecoderOnlyInputs, token_inputs
14 from vllm.model_executor.layers.activation import get_act_fn
15 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 QKVParallelLinear,
@@ -63,7 +63,7 @@ def dummy_seq_data_for_blip(
63 else:
64 image_feature_size = image_feature_size_override
65
66 - return SequenceData.from_token_counts(
66 + return SequenceData.from_prompt_token_counts(
67 (image_token_id, image_feature_size * num_images),
68 (0, seq_len - image_feature_size * num_images),
69 )
@@ -89,14 +89,14 @@ def dummy_image_for_blip(
89 def input_processor_for_blip(
90 model_config: ModelConfig,
91 hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
92 - llm_inputs: LLMInputs,
92 + inputs: DecoderOnlyInputs,
93 *,
94 image_token_id: int,
95 image_feature_size_override: Optional[int] = None,
96 ):
97 - multi_modal_data = llm_inputs.get("multi_modal_data")
97 + multi_modal_data = inputs.get("multi_modal_data")
98 if multi_modal_data is None or "image" not in multi_modal_data:
99 - return llm_inputs
99 + return inputs
100
101 tokenizer = cached_get_tokenizer(model_config.tokenizer)
102
@@ -107,16 +107,16 @@ def input_processor_for_blip(
107
108 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
109 tokenizer,
110 - llm_inputs.get("prompt"),
111 - llm_inputs["prompt_token_ids"],
110 + inputs.get("prompt"),
111 + inputs["prompt_token_ids"],
112 placeholder_token_id=image_token_id,
113 repeat_count=image_feature_size,
114 )
115
116 # NOTE: Create a defensive copy of the original inputs
117 - return LLMInputs(prompt_token_ids=new_token_ids,
118 - prompt=new_prompt,
119 - multi_modal_data=multi_modal_data)
117 + return token_inputs(prompt_token_ids=new_token_ids,
118 + prompt=new_prompt,
119 + multi_modal_data=multi_modal_data)
120
121
122 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
vllm/model_executor/models/blip2.py
+11 -10
@@ -9,7 +9,8 @@ from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
9
10 from vllm.attention import AttentionMetadata
11 from vllm.config import CacheConfig, MultiModalConfig
12 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
13 + token_inputs)
14 from vllm.model_executor.layers.activation import get_act_fn
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -421,7 +422,7 @@ def dummy_seq_data_for_blip2(
422 else:
423 image_feature_size = image_feature_size_override
424
424 - return SequenceData.from_token_counts(
425 + return SequenceData.from_prompt_token_counts(
426 (image_token_id, image_feature_size * num_images),
427 (0, seq_len - image_feature_size * num_images),
428 )
@@ -449,10 +450,10 @@ def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
450 raise NotImplementedError(msg)
451
452
452 -def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
453 - multi_modal_data = llm_inputs.get("multi_modal_data")
453 +def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
454 + multi_modal_data = inputs.get("multi_modal_data")
455 if multi_modal_data is None or "image" not in multi_modal_data:
455 - return llm_inputs
456 + return inputs
457
458 hf_config = ctx.get_hf_config(Blip2Config)
459 image_feature_size = get_blip2_image_feature_size(hf_config)
@@ -460,15 +461,15 @@ def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
461 # The original model places image tokens at the front
462 # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514
463 new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
463 - new_token_ids += llm_inputs["prompt_token_ids"]
464 + new_token_ids += inputs["prompt_token_ids"]
465
465 - new_prompt = llm_inputs.get("prompt")
466 + new_prompt = inputs.get("prompt")
467 if new_prompt is not None:
468 new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt
469
469 - return LLMInputs(prompt_token_ids=new_token_ids,
470 - prompt=new_prompt,
471 - multi_modal_data=multi_modal_data)
470 + return token_inputs(prompt_token_ids=new_token_ids,
471 + prompt=new_prompt,
472 + multi_modal_data=multi_modal_data)
473
474
475 @MULTIMODAL_REGISTRY.register_image_input_mapper()
vllm/model_executor/models/chameleon.py
+12 -10
@@ -11,7 +11,8 @@ from transformers import ChameleonConfig, ChameleonVQVAEConfig
11 from vllm.attention import Attention, AttentionMetadata
12 from vllm.config import CacheConfig, MultiModalConfig
13 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15 + token_inputs)
16 from vllm.model_executor.layers.activation import SiluAndMul
17 from vllm.model_executor.layers.layernorm import RMSNorm
18 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -69,7 +70,7 @@ def dummy_seq_data_for_chameleon(
70 else:
71 image_feature_size = image_feature_size_override
72
72 - return SequenceData.from_token_counts(
73 + return SequenceData.from_prompt_token_counts(
74 (image_token_id, image_feature_size * num_images),
75 (0, seq_len - image_feature_size * num_images),
76 )
@@ -106,7 +107,8 @@ def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
107 return seq_data, mm_data
108
109
109 -def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
110 +def input_processor_for_chameleon(ctx: InputContext,
111 + inputs: DecoderOnlyInputs):
112
113 """
114 Processing input prompt to insert required tokens for image placeholder.
@@ -114,16 +116,16 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
116 See https://github.com/huggingface/transformers/blob/0fdea8607d7e01eb0e38a1ebeb7feee30a22f0cf/src/transformers/models/chameleon/processing_chameleon.py#L58
117 """ # noqa
118
117 - multi_modal_data = llm_inputs.get("multi_modal_data")
119 + multi_modal_data = inputs.get("multi_modal_data")
120 if multi_modal_data is None or "image" not in multi_modal_data:
119 - return llm_inputs
121 + return inputs
122
123 model_config = ctx.model_config
124 tokenizer = cached_get_tokenizer(model_config.tokenizer)
125 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
126 tokenizer,
125 - llm_inputs.get("prompt"),
126 - llm_inputs["prompt_token_ids"],
127 + inputs.get("prompt"),
128 + inputs["prompt_token_ids"],
129 placeholder_token_id=CHAMELEON_IMAGE_TOKEN_ID,
130 repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
131 pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
@@ -137,9 +139,9 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
139 new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
140
141 # NOTE: Create a defensive copy of the original inputs
140 - return LLMInputs(prompt_token_ids=new_token_ids,
141 - prompt=new_prompt,
142 - multi_modal_data=multi_modal_data)
142 + return token_inputs(prompt_token_ids=new_token_ids,
143 + prompt=new_prompt,
144 + multi_modal_data=multi_modal_data)
145
146
147 class ChameleonLayerNorm(nn.LayerNorm):
vllm/model_executor/models/chatglm.py
+11 -11
@@ -14,7 +14,7 @@ from torch.nn import LayerNorm
14 from vllm.attention import Attention, AttentionMetadata
15 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16 from vllm.distributed import get_tensor_model_parallel_world_size
17 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
17 +from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
18 from vllm.logger import init_logger
19 from vllm.model_executor.layers.activation import SiluAndMul
20 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -149,20 +149,20 @@ def find_all_positions(input_ids: List[int], target: int) -> List[int]:
149 return [index for index, value in enumerate(input_ids) if value == target]
150
151
152 -def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
152 +def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
153 hf_config = ctx.get_hf_config(ChatGLMConfig)
154 vision_config = getattr(hf_config, 'vision_config', None)
155
156 if vision_config is None:
157 - return llm_inputs
157 + return inputs
158 elif isinstance(vision_config, dict):
159 image_placeholder_length = calculate_image_placeholder(vision_config)
160 else:
161 msg = f"Unsupported vision config: {type(vision_config)}"
162 raise NotImplementedError(msg)
163
164 - input_ids = llm_inputs.get("prompt_token_ids")
165 - position_ids = llm_inputs.get("position_ids")
164 + input_ids = inputs.get("prompt_token_ids")
165 + position_ids = inputs.get("position_ids")
166 tokenizer = cached_get_tokenizer(
167 ctx.model_config.model,
168 trust_remote_code=ctx.model_config.trust_remote_code)
@@ -171,15 +171,15 @@ def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
171 raw_batch_data = tokenizer.apply_chat_template(
172 conversation=[{
173 "role": "user",
174 - "image": llm_inputs['multi_modal_data']["image"],
175 - "content": llm_inputs['prompt']
174 + "image": inputs['multi_modal_data']["image"],
175 + "content": inputs['prompt']
176 }],
177 add_generation_prompt=True,
178 tokenize=True,
179 return_tensors="pt",
180 return_dict=True).data
181 except Exception:
182 - logger.error("Failed to process content (%s)", llm_inputs['prompt'])
182 + logger.error("Failed to process content (%s)", inputs['prompt'])
183 raise
184 input_ids = raw_batch_data['input_ids'][0].tolist()
185
@@ -214,9 +214,9 @@ def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
214
215 assert len(new_input_ids) == len(new_position_ids)
216
217 - llm_inputs["prompt_token_ids"] = new_input_ids
218 - llm_inputs["position_ids"] = new_position_ids
219 - return llm_inputs
217 + inputs["prompt_token_ids"] = new_input_ids
218 + inputs["position_ids"] = new_position_ids
219 + return inputs
220
221
222 class GLMAttention(nn.Module):
vllm/model_executor/models/clip.py
+10 -10
@@ -11,7 +11,7 @@ from transformers.models.clip.modeling_clip import CLIPSdpaAttention
11
12 from vllm.config import ModelConfig
13 from vllm.distributed import divide, get_tensor_model_parallel_world_size
14 -from vllm.inputs import LLMInputs
14 +from vllm.inputs import DecoderOnlyInputs, token_inputs
15 from vllm.model_executor.layers.activation import get_act_fn
16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 QKVParallelLinear,
@@ -62,7 +62,7 @@ def dummy_seq_data_for_clip(
62 else:
63 image_feature_size = image_feature_size_override
64
65 - return SequenceData.from_token_counts(
65 + return SequenceData.from_prompt_token_counts(
66 (image_token_id, image_feature_size * num_images),
67 (0, seq_len - image_feature_size * num_images),
68 )
@@ -106,14 +106,14 @@ def dummy_video_for_clip(
106 def input_processor_for_clip(
107 model_config: ModelConfig,
108 hf_config: CLIPVisionConfig,
109 - llm_inputs: LLMInputs,
109 + inputs: DecoderOnlyInputs,
110 *,
111 image_token_id: int,
112 image_feature_size_override: Optional[Union[int, List[int]]] = None,
113 ):
114 - multi_modal_data = llm_inputs.get("multi_modal_data")
114 + multi_modal_data = inputs.get("multi_modal_data")
115 if multi_modal_data is None or "image" not in multi_modal_data:
116 - return llm_inputs
116 + return inputs
117
118 tokenizer = cached_get_tokenizer(model_config.tokenizer)
119
@@ -130,16 +130,16 @@ def input_processor_for_clip(
130
131 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
132 tokenizer,
133 - llm_inputs.get("prompt"),
134 - llm_inputs["prompt_token_ids"],
133 + inputs.get("prompt"),
134 + inputs["prompt_token_ids"],
135 placeholder_token_id=image_token_id,
136 repeat_count=image_feature_size,
137 )
138
139 # NOTE: Create a defensive copy of the original inputs
140 - return LLMInputs(prompt_token_ids=new_token_ids,
141 - prompt=new_prompt,
142 - multi_modal_data=multi_modal_data)
140 + return token_inputs(prompt_token_ids=new_token_ids,
141 + prompt=new_prompt,
142 + multi_modal_data=multi_modal_data)
143
144
145 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
vllm/model_executor/models/fuyu.py
+10 -9
@@ -27,7 +27,8 @@ from transformers import FuyuConfig, FuyuImageProcessor
27
28 from vllm.attention import AttentionMetadata
29 from vllm.config import CacheConfig, MultiModalConfig
30 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
31 + token_inputs)
32 from vllm.model_executor.layers.linear import ColumnParallelLinear
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -149,10 +150,10 @@ def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
150 return model_image_input
151
152
152 -def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
153 - multi_modal_data = llm_inputs.get("multi_modal_data")
153 +def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
154 + multi_modal_data = inputs.get("multi_modal_data")
155 if multi_modal_data is None or "image" not in multi_modal_data:
155 - return llm_inputs
156 + return inputs
157
158 model_config = ctx.model_config
159 image_data = multi_modal_data["image"]
@@ -176,8 +177,8 @@ def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
177 raise TypeError(f"Invalid image type: {type(image_data)}")
178
179 # process prompts
179 - prompt = llm_inputs.get("prompt")
180 - prompt_token_ids = llm_inputs["prompt_token_ids"]
180 + prompt = inputs.get("prompt")
181 + prompt_token_ids = inputs["prompt_token_ids"]
182 tokenizer = cached_get_tokenizer(model_config.model)
183 # dim0 is batch_size, dim1 is subseq_size which will always be 1
184 image_input_ids: List[List[
@@ -190,9 +191,9 @@ def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
191 new_prompt_token_ids = image_input_ids + bos_token + prompt_token_ids[
192 1:] + boa_token
193
193 - return LLMInputs(prompt=new_prompt,
194 - prompt_token_ids=new_prompt_token_ids,
195 - multi_modal_data=new_multi_modal_data)
194 + return token_inputs(prompt=new_prompt,
195 + prompt_token_ids=new_prompt_token_ids,
196 + multi_modal_data=new_multi_modal_data)
197
198
199 def input_mapper_for_fuyu(ctx: InputContext, data: object):
vllm/model_executor/models/internvl.py
+11 -10
@@ -17,7 +17,8 @@ from transformers import PretrainedConfig
17
18 from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, MultiModalConfig
20 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
20 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21 + token_inputs)
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24 from vllm.model_executor.models.intern_vit import InternVisionModel
@@ -276,13 +277,13 @@ class InternVLInputPipeline:
277 def input_processor(
278 self,
279 ctx: InputContext,
279 - llm_inputs: LLMInputs,
280 + inputs: DecoderOnlyInputs,
281 *,
282 max_dynamic_patch: Optional[int] = None,
282 - ) -> LLMInputs:
283 - multi_modal_data = llm_inputs.get("multi_modal_data")
283 + ) -> DecoderOnlyInputs:
284 + multi_modal_data = inputs.get("multi_modal_data")
285 if multi_modal_data is None or "image" not in multi_modal_data:
285 - return llm_inputs
286 + return inputs
287
288 model_config = ctx.model_config
289 hf_config = ctx.get_hf_config()
@@ -311,8 +312,8 @@ class InternVLInputPipeline:
312 model_config.tokenizer,
313 trust_remote_code=model_config.trust_remote_code)
314
314 - prompt = llm_inputs.get("prompt")
315 - prompt_token_ids = llm_inputs["prompt_token_ids"]
315 + prompt = inputs.get("prompt")
316 + prompt_token_ids = inputs["prompt_token_ids"]
317 if prompt is None:
318 prompt = tokenizer.decode(prompt_token_ids)
319
@@ -320,9 +321,9 @@ class InternVLInputPipeline:
321 num_patches)
322 new_prompt_token_ids = tokenizer.encode(new_prompt)
323
323 - return LLMInputs(prompt=prompt,
324 - prompt_token_ids=new_prompt_token_ids,
325 - multi_modal_data=multi_modal_data)
324 + return token_inputs(prompt=prompt,
325 + prompt_token_ids=new_prompt_token_ids,
326 + multi_modal_data=multi_modal_data)
327
328 def input_mapper(
329 self,
vllm/model_executor/models/llava.py
+6 -6
@@ -9,7 +9,7 @@ from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
9
10 from vllm.attention import AttentionMetadata
11 from vllm.config import CacheConfig, MultiModalConfig
12 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12 +from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
13 from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.quantization import QuantizationConfig
15 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -125,10 +125,10 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
125 raise NotImplementedError(msg)
126
127
128 -def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
129 - multi_modal_data = llm_inputs.get("multi_modal_data")
128 +def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
129 + multi_modal_data = inputs.get("multi_modal_data")
130 if multi_modal_data is None or "image" not in multi_modal_data:
131 - return llm_inputs
131 + return inputs
132
133 model_config = ctx.model_config
134 hf_config = ctx.get_hf_config(LlavaConfig)
@@ -151,7 +151,7 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
151 return input_processor_for_clip(
152 model_config,
153 vision_config,
154 - llm_inputs,
154 + inputs,
155 image_token_id=hf_config.image_token_index,
156 image_feature_size_override=image_feature_size,
157 )
@@ -159,7 +159,7 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
159 return input_processor_for_siglip(
160 model_config,
161 vision_config,
162 - llm_inputs,
162 + inputs,
163 image_token_id=hf_config.image_token_index,
164 image_feature_size_override=image_feature_size,
165 )
vllm/model_executor/models/llava_next.py
+7 -6
@@ -12,7 +12,7 @@ from typing_extensions import NotRequired
12
13 from vllm.attention import AttentionMetadata
14 from vllm.config import CacheConfig, MultiModalConfig
15 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15 +from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -201,10 +201,11 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
201 raise NotImplementedError(msg)
202
203
204 -def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
205 - multi_modal_data = llm_inputs.get("multi_modal_data")
204 +def input_processor_for_llava_next(ctx: InputContext,
205 + inputs: DecoderOnlyInputs):
206 + multi_modal_data = inputs.get("multi_modal_data")
207 if multi_modal_data is None or "image" not in multi_modal_data:
207 - return llm_inputs
208 + return inputs
209
210 model_config = ctx.model_config
211 hf_config = ctx.get_hf_config(LlavaNextConfig)
@@ -239,7 +240,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
240 return input_processor_for_clip(
241 model_config,
242 vision_config,
242 - llm_inputs,
243 + inputs,
244 image_token_id=hf_config.image_token_index,
245 image_feature_size_override=image_feature_size,
246 )
@@ -247,7 +248,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
248 return input_processor_for_siglip(
249 model_config,
250 vision_config,
250 - llm_inputs,
251 + inputs,
252 image_token_id=hf_config.image_token_index,
253 image_feature_size_override=image_feature_size,
254 )
vllm/model_executor/models/llava_next_video.py
+10 -9
@@ -11,7 +11,8 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
11
12 from vllm.attention import AttentionMetadata
13 from vllm.config import CacheConfig, MultiModalConfig
14 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15 + token_inputs)
16 from vllm.model_executor.layers.activation import get_act_fn
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -139,10 +140,10 @@ def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
140
141
142 def input_processor_for_llava_next_video(ctx: InputContext,
142 - llm_inputs: LLMInputs):
143 - multi_modal_data = llm_inputs.get("multi_modal_data")
143 + inputs: DecoderOnlyInputs):
144 + multi_modal_data = inputs.get("multi_modal_data")
145 if multi_modal_data is None or "video" not in multi_modal_data:
145 - return llm_inputs
146 + return inputs
147 video_data = multi_modal_data["video"]
148
149 model_config = ctx.model_config
@@ -160,15 +161,15 @@ def input_processor_for_llava_next_video(ctx: InputContext,
161
162 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
163 tokenizer,
163 - llm_inputs.get("prompt"),
164 - llm_inputs["prompt_token_ids"],
164 + inputs.get("prompt"),
165 + inputs["prompt_token_ids"],
166 placeholder_token_id=hf_config.video_token_index,
167 repeat_count=video_feature_size,
168 )
169
169 - return LLMInputs(prompt_token_ids=new_token_ids,
170 - prompt=new_prompt,
171 - multi_modal_data=multi_modal_data)
170 + return token_inputs(prompt_token_ids=new_token_ids,
171 + prompt=new_prompt,
172 + multi_modal_data=multi_modal_data)
173
174 elif is_list_of(video_data, np.ndarray):
175 raise NotImplementedError(
vllm/model_executor/models/llava_onevision.py
+20 -22
@@ -15,8 +15,8 @@ from typing_extensions import NotRequired
15
16 from vllm.attention import AttentionMetadata
17 from vllm.config import CacheConfig, MultiModalConfig
18 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
19 -from vllm.logger import init_logger
18 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
19 + token_inputs)
20 from vllm.model_executor.layers.activation import get_act_fn
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -37,8 +37,6 @@ from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
37 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38 merge_multimodal_embeddings)
39
40 -logger = init_logger(__name__)
41 -
40 # Result in the max possible feature size (2x2 grid of 336x336px tiles)
41 MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
42
@@ -252,10 +250,10 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
250
251
252 def input_processor_when_multimodal_input_image(ctx: InputContext,
255 - llm_inputs: LLMInputs):
256 - multi_modal_data = llm_inputs.get("multi_modal_data")
253 + inputs: DecoderOnlyInputs):
254 + multi_modal_data = inputs.get("multi_modal_data")
255 if multi_modal_data is None or "image" not in multi_modal_data:
258 - return llm_inputs
256 + return inputs
257
258 model_config = ctx.model_config
259 hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
@@ -290,7 +288,7 @@ def input_processor_when_multimodal_input_image(ctx: InputContext,
288 return input_processor_for_clip(
289 model_config,
290 vision_config,
293 - llm_inputs,
291 + inputs,
292 image_token_id=hf_config.image_token_index,
293 image_feature_size_override=image_feature_size,
294 )
@@ -298,7 +296,7 @@ def input_processor_when_multimodal_input_image(ctx: InputContext,
296 return input_processor_for_siglip(
297 model_config,
298 vision_config,
301 - llm_inputs,
299 + inputs,
300 image_token_id=hf_config.image_token_index,
301 image_feature_size_override=image_feature_size,
302 )
@@ -308,10 +306,10 @@ def input_processor_when_multimodal_input_image(ctx: InputContext,
306
307
308 def input_processor_when_multimodal_input_video(ctx: InputContext,
311 - llm_inputs: LLMInputs):
312 - multi_modal_data = llm_inputs.get("multi_modal_data")
309 + inputs: DecoderOnlyInputs):
310 + multi_modal_data = inputs.get("multi_modal_data")
311 if multi_modal_data is None or "video" not in multi_modal_data:
314 - return llm_inputs
312 + return inputs
313 video_data = multi_modal_data["video"]
314
315 model_config = ctx.model_config
@@ -326,15 +324,15 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
324
325 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
326 tokenizer,
329 - llm_inputs.get("prompt"),
330 - llm_inputs["prompt_token_ids"],
327 + inputs.get("prompt"),
328 + inputs["prompt_token_ids"],
329 placeholder_token_id=hf_config.video_token_index,
330 repeat_count=video_feature_size,
331 )
332
335 - return LLMInputs(prompt_token_ids=new_token_ids,
336 - prompt=new_prompt,
337 - multi_modal_data=multi_modal_data)
333 + return token_inputs(prompt_token_ids=new_token_ids,
334 + prompt=new_prompt,
335 + multi_modal_data=multi_modal_data)
336
337 elif is_list_of(video_data, np.ndarray):
338 raise NotImplementedError(
@@ -345,15 +343,15 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
343
344
345 def input_processor_for_llava_onevision(ctx: InputContext,
348 - llm_inputs: LLMInputs):
349 - multi_modal_data = llm_inputs.get("multi_modal_data")
346 + inputs: DecoderOnlyInputs):
347 + multi_modal_data = inputs.get("multi_modal_data")
348 if multi_modal_data is None or ("video" not in multi_modal_data
349 and "image" not in multi_modal_data):
352 - return llm_inputs
350 + return inputs
351 if "image" in multi_modal_data:
354 - return input_processor_when_multimodal_input_image(ctx, llm_inputs)
352 + return input_processor_when_multimodal_input_image(ctx, inputs)
353 if "video" in multi_modal_data:
356 - return input_processor_when_multimodal_input_video(ctx, llm_inputs)
354 + return input_processor_when_multimodal_input_video(ctx, inputs)
355
356 msg = "Unsupported multi data type"
357 raise NotImplementedError(msg)
vllm/model_executor/models/minicpmv.py
+9 -9
@@ -36,7 +36,8 @@ from typing_extensions import NotRequired
36
37 from vllm.attention import AttentionMetadata
38 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
39 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
39 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
40 + token_inputs)
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.quantization import QuantizationConfig
43 from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
@@ -256,7 +257,7 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
257
258
259 def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
259 - return SequenceData.from_token_counts((0, seq_len))
260 + return SequenceData.from_prompt_token_counts((0, seq_len))
261
262
263 def dummy_image_for_minicpmv(ctx: InputContext, hf_config: PretrainedConfig,
@@ -279,10 +280,10 @@ def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
280 return seq_data, mm_data
281
282
282 -def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
283 - multi_modal_data = llm_inputs.get("multi_modal_data")
283 +def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
284 + multi_modal_data = inputs.get("multi_modal_data")
285 if multi_modal_data is None or "image" not in multi_modal_data:
285 - return llm_inputs
286 + return inputs
287 model_config = ctx.model_config
288 version = get_version_by_config(model_config.hf_config)
289 tokenizer = cached_get_tokenizer(
@@ -297,8 +298,8 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
298 return image_processor. \
299 get_slice_image_placeholder(image_size, num_image)
300
300 - prompt = llm_inputs.get("prompt")
301 - token_ids = llm_inputs.get("prompt_token_ids")
301 + prompt = inputs.get("prompt")
302 + token_ids = inputs.get("prompt_token_ids")
303 if prompt is None:
304 prompt = tokenizer.decode(token_ids)
305
@@ -332,12 +333,11 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
333 _build_image_input(ctx, image) for image in images
334 ]
335
335 - llm_inputs = LLMInputs(
336 + return token_inputs(
337 prompt_token_ids=new_token_ids,
338 prompt=new_prompt,
339 multi_modal_data=multi_modal_data,
340 )
340 - return llm_inputs
341
342
343 def input_mapper_for_minicpmv(ctx: InputContext, data: object):
vllm/model_executor/models/mllama.py
+26 -26
@@ -14,7 +14,6 @@
14 # limitations under the License.
15 """PyTorch Mllama model."""
16 import math
17 -from array import array
17 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
18 TypedDict, Union)
19
@@ -37,7 +36,8 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
36 from vllm.attention.ops.paged_attn import PagedAttention
37 from vllm.config import CacheConfig, MultiModalConfig
38 from vllm.distributed import get_tensor_model_parallel_world_size
40 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
39 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
40 + EncoderDecoderInputs, InputContext)
41 from vllm.logger import init_logger
42 from vllm.model_executor.layers.layernorm import RMSNorm
43 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -51,7 +51,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
51 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52 from vllm.model_executor.sampling_metadata import SamplingMetadata
53 from vllm.multimodal import MULTIMODAL_REGISTRY
54 -from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
54 +from vllm.sequence import SequenceData
55
56 from .clip import CLIPMLP
57 from .interfaces import SupportsMultiModal
@@ -86,24 +86,24 @@ def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
86 return num_images
87
88
89 -def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
89 +def input_processor_for_mllama(ctx: InputContext,
90 + inputs: Union[DecoderOnlyInputs,
91 + EncoderDecoderInputs]):
92 # move encoder_prompt to prompt
91 - if llm_inputs.get("prompt") is None:
92 - llm_inputs["prompt"] = llm_inputs["encoder_prompt"]
93 - llm_inputs["prompt_token_ids"] = llm_inputs["encoder_prompt_token_ids"]
93 + if inputs.get("prompt") is None:
94 + inputs["prompt"] = inputs["encoder_prompt"]
95 + inputs["prompt_token_ids"] = inputs["encoder_prompt_token_ids"]
96
97 # process multi-modal data
96 - assert "decoder_multi_modal_data" not in llm_inputs, \
97 - "multi-modal data should be put in encoder message of mllama"
98 - multi_modal_data = llm_inputs.get("encoder_multi_modal_data")
98 + multi_modal_data = inputs.get("encoder_multi_modal_data")
99
100 if multi_modal_data is None or "image" not in multi_modal_data \
101 or multi_modal_data["image"] is None:
102 # text-only
103 - llm_inputs["encoder_prompt"] = ""
104 - llm_inputs["encoder_prompt_token_ids"] = []
105 - llm_inputs["encoder_multi_modal_data"] = {}
106 - return llm_inputs
103 + inputs["encoder_prompt"] = ""
104 + inputs["encoder_prompt_token_ids"] = []
105 + inputs["encoder_multi_modal_data"] = {}
106 + return inputs
107
108 if isinstance(multi_modal_data['image'], Image.Image):
109 multi_modal_data['image'] = [multi_modal_data['image']]
@@ -111,7 +111,7 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
111 # are attended by the decoded tokens, we only need to
112 # get the number of tiles for those images.
113 num_decode_images = _get_num_image_in_last_group(
114 - llm_inputs["prompt_token_ids"])
114 + inputs["prompt_token_ids"])
115 hf_config = ctx.model_config.hf_config
116 num_tiles = 0
117 for image in multi_modal_data["image"][::-1]:
@@ -137,11 +137,10 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
137 "chunk size should be multiple of 14"
138 token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
139 num_tokens = num_tiles * token_per_chunk
140 - llm_inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
141 - llm_inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID
142 - ] * num_tokens
140 + inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
141 + inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID] * num_tokens
142
144 - return llm_inputs
143 + return inputs
144
145
146 def get_max_mllama_image_tokens(ctx: InputContext) -> int:
@@ -154,17 +153,18 @@ def dummy_decoder_seq_data(seq_len: int, num_images: int):
153 # <|image|> * num_images + 0 * (seq_len - num_images)
154 assert seq_len >= num_images, \
155 "seq_len should be greater than or equal to num_images"
157 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
158 - [MLLAMA_IMAGE_TOKEN_ID]) * num_images
159 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * (seq_len - num_images)
160 - return SequenceData(token_ids)
156 +
157 + return SequenceData.from_prompt_token_counts(
158 + (MLLAMA_IMAGE_TOKEN_ID, num_images),
159 + (0, seq_len - num_images),
160 + )
161
162
163 def dummy_encoder_seq_data(ctx: InputContext, num_images: int):
164 num_tokens = get_max_mllama_image_tokens(ctx) * num_images
165 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
166 - [MLLAMA_IMAGE_TOKEN_ID]) * num_tokens
167 - return SequenceData(token_ids)
165 +
166 + return SequenceData.from_prompt_token_counts(
167 + (MLLAMA_IMAGE_TOKEN_ID, num_tokens))
168
169
170 def dummy_image(num_images: int, ):
vllm/model_executor/models/molmo.py
+8 -9
@@ -23,7 +23,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
23 get_tensor_model_parallel_world_size,
24 split_tensor_along_last_dim,
25 tensor_model_parallel_all_gather)
26 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
26 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
27 + token_inputs)
28 from vllm.model_executor import SamplingMetadata
29 from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
30 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -945,9 +946,9 @@ def pad_images(
946 return images, image_input_idx, image_masks
947
948
948 -def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
949 - prompt = llm_inputs.get("prompt", None)
950 - multi_modal_data = llm_inputs.get("multi_modal_data", None)
949 +def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
950 + prompt = inputs.get("prompt", None)
951 + multi_modal_data = inputs.get("multi_modal_data", None)
952 if multi_modal_data is not None:
953 image = multi_modal_data.get("image", None)
954 else:
@@ -965,9 +966,7 @@ def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
966 elif prompt is not None:
967 out = processor.process(prompt, image)
968 else:
968 - out = processor.process(None,
969 - image,
970 - tokens=llm_inputs["prompt_token_ids"])
969 + out = processor.process(None, image, tokens=inputs["prompt_token_ids"])
970
971 image_processor = processor.image_processor
972 max_total_crops = 1 + image_processor.max_crops
@@ -1020,9 +1019,9 @@ def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
1019
1020 multi_modal_data = dict(image=image_data)
1021
1023 - return LLMInputs(
1022 + return token_inputs(
1023 prompt_token_ids=out["input_ids"],
1025 - prompt=llm_inputs["prompt"],
1024 + prompt=inputs["prompt"],
1025 multi_modal_data=multi_modal_data,
1026 )
1027
vllm/model_executor/models/paligemma.py
+11 -9
@@ -7,7 +7,8 @@ from transformers import PaliGemmaConfig
7
8 from vllm.attention import AttentionMetadata
9 from vllm.config import CacheConfig, MultiModalConfig
10 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
10 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
11 + token_inputs)
12 from vllm.logger import init_logger
13 from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -68,7 +69,8 @@ def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
69 return seq_data, mm_data
70
71
71 -def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
72 +def input_processor_for_paligemma(ctx: InputContext,
73 + inputs: DecoderOnlyInputs):
74
75 """
76 The correct prompt format needs to be:
@@ -77,9 +79,9 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
79 See https://github.com/huggingface/transformers/blob/25245ec26dc29bcf6102e1b4ddd0dfd02e720cf5/src/transformers/models/paligemma/processing_paligemma.py#L55
80 """ # noqa
81
80 - multi_modal_data = llm_inputs.get("multi_modal_data")
82 + multi_modal_data = inputs.get("multi_modal_data")
83 if multi_modal_data is None or "image" not in multi_modal_data:
82 - return llm_inputs
84 + return inputs
85
86 model_config = ctx.model_config
87 hf_config = ctx.get_hf_config(PaliGemmaConfig)
@@ -91,8 +93,8 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
93 image_token_str_pad = image_token_str * image_feature_size
94 image_token_ids_pad = [hf_config.image_token_index] * image_feature_size
95
94 - orig_prompt = llm_inputs.get("prompt")
95 - orig_prompt_ids = llm_inputs.get("prompt_token_ids")
96 + orig_prompt = inputs.get("prompt")
97 + orig_prompt_ids = inputs.get("prompt_token_ids")
98
99 if orig_prompt is not None and image_token_str in orig_prompt:
100 logger.warning(
@@ -106,9 +108,9 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
108 new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline
109
110 # NOTE: Create a defensive copy of the original inputs
109 - return LLMInputs(prompt_token_ids=new_token_ids,
110 - prompt=new_prompt,
111 - multi_modal_data=multi_modal_data)
111 + return token_inputs(prompt_token_ids=new_token_ids,
112 + prompt=new_prompt,
113 + multi_modal_data=multi_modal_data)
114
115
116 class PaliGemmaMultiModalProjector(nn.Module):
vllm/model_executor/models/phi3v.py
+10 -10
@@ -27,7 +27,8 @@ from transformers import CLIPVisionConfig, PretrainedConfig
27
28 from vllm.attention import AttentionMetadata
29 from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
30 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
31 + token_inputs)
32 from vllm.logger import init_logger
33 from vllm.model_executor.layers.pooler import Pooler, PoolingType
34 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -410,12 +411,12 @@ def _get_image_placeholder_token_id_candidates(
411
412
413 def input_processor_for_phi3v(ctx: InputContext,
413 - llm_inputs: LLMInputs,
414 + inputs: DecoderOnlyInputs,
415 *,
416 num_crops: Optional[int] = None):
416 - multi_modal_data = llm_inputs.get("multi_modal_data")
417 + multi_modal_data = inputs.get("multi_modal_data")
418 if multi_modal_data is None or "image" not in multi_modal_data:
418 - return llm_inputs
419 + return inputs
420
421 model_config = ctx.model_config
422 hf_config = ctx.get_hf_image_processor_config()
@@ -447,7 +448,7 @@ def input_processor_for_phi3v(ctx: InputContext,
448 else:
449 raise TypeError(f"Invalid image type: {type(image_data)}")
450
450 - prompt = llm_inputs.get("prompt")
451 + prompt = inputs.get("prompt")
452 if prompt is None:
453 # for async server request, we assume prompt and its token_ids is always
454 # in correct format. And num_image_tags == len(image_data) always True.
@@ -464,7 +465,7 @@ def input_processor_for_phi3v(ctx: InputContext,
465 image_data), "The count of image_placeholder not match image's"
466 new_prompt = prompt
467
467 - prompt_token_ids = llm_inputs["prompt_token_ids"].copy()
468 + prompt_token_ids = inputs["prompt_token_ids"].copy()
469
470 print("prompt_token_ids (old)", prompt_token_ids)
471
@@ -506,10 +507,9 @@ def input_processor_for_phi3v(ctx: InputContext,
507 new_token_ids.append(token_id)
508
509 # NOTE: Create a defensive copy of the original inputs
509 - llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
510 - prompt=new_prompt,
511 - multi_modal_data=multi_modal_data)
512 - return llm_inputs
510 + return token_inputs(prompt_token_ids=new_token_ids,
511 + prompt=new_prompt,
512 + multi_modal_data=multi_modal_data)
513
514
515 @MULTIMODAL_REGISTRY.register_image_input_mapper()
vllm/model_executor/models/pixtral.py
+7 -7
@@ -14,7 +14,7 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalMask
14
15 from vllm.attention import AttentionMetadata
16 from vllm.config import CacheConfig, MultiModalConfig
17 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
17 +from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
18 from vllm.model_executor.layers.layernorm import RMSNorm
19 from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -62,7 +62,7 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
62 image_feature_size = (size**2) // (patch_size**2)
63
64 num_image_tokens = image_feature_size * num_images
65 - seq_data = SequenceData.from_token_counts(
65 + seq_data = SequenceData.from_prompt_token_counts(
66 (image_token_id, num_image_tokens),
67 (0, seq_len - num_image_tokens),
68 )
@@ -102,8 +102,8 @@ def input_mapper_for_pixtral(ctx: InputContext,
102 return MultiModalInputs({"images": images})
103
104
105 -def input_processor_for_pixtral(ctx: InputContext, llm_inputs: LLMInputs):
106 - multi_modal_data = llm_inputs.get("multi_modal_data")
105 +def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
106 + multi_modal_data = inputs.get("multi_modal_data")
107 if multi_modal_data is not None and "image" in multi_modal_data:
108 tokenizer = cached_get_tokenizer(
109 ctx.model_config.tokenizer,
@@ -112,15 +112,15 @@ def input_processor_for_pixtral(ctx: InputContext, llm_inputs: LLMInputs):
112 mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
113 image_token_id = mm_encoder.special_ids.img
114
115 - if image_token_id not in llm_inputs['prompt_token_ids']:
115 + if image_token_id not in inputs['prompt_token_ids']:
116 raise ValueError(
117 - (f"You've passed {llm_inputs=} without {image_token_id=}"
117 + (f"You've passed {inputs=} without {image_token_id=}"
118 " Make sure to process your input via mistral_common's"
119 " tokenizer or pass a chat completion request. For more"
120 " For more info, see: "
121 "https://github.com/vllm-project/vllm/issues/8411."))
122
123 - return llm_inputs
123 + return inputs
124
125
126 @MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral)
vllm/model_executor/models/qwen.py
+13 -12
@@ -22,7 +22,8 @@ from transformers import PretrainedConfig
22 from vllm.attention import Attention, AttentionMetadata
23 from vllm.config import CacheConfig, MultiModalConfig
24 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
25 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
25 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
26 + token_inputs)
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
29 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -652,30 +653,30 @@ def get_image_text(image_num: int, padding: bool) -> str:
653
654
655 def input_processor_for_qwen(ctx: InputContext,
655 - llm_inputs: LLMInputs) -> LLMInputs:
656 + inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
657 """Processes the inputs, which may or may not be multimodal.
658 Multimodal inputs will only be processed if the model has a "visual"
659 component in its model config, otherwise they'll be ignored.
660
661 Args:
662 ctx: Context of the loaded model.
662 - llm_inputs: LLM inputs which may have a multi_modal_data attribute.
663 + inputs: LLM inputs which may have a multi_modal_data attribute.
664
665 Returns:
666 If the model is language only or not multimodal inputs were provided,
666 - returns llm_inputs unmodified. Otherwise, processes the multimodal
667 + returns inputs unmodified. Otherwise, processes the multimodal
668 images / image embeddings and adds the fixed-length image placeholders.
669 """
669 - multi_modal_data = llm_inputs.get("multi_modal_data")
670 + multi_modal_data = inputs.get("multi_modal_data")
671
672 # Only process images if we have multimodal data and a visual config
673 hf_config = ctx.get_hf_config()
674 if (multi_modal_data is None or "image" not in multi_modal_data
675 or not hasattr(hf_config, "visual")):
675 - return llm_inputs
676 + return inputs
677
677 - prompt = llm_inputs.get("prompt")
678 - prompt_token_ids = llm_inputs["prompt_token_ids"]
678 + prompt = inputs.get("prompt")
679 + prompt_token_ids = inputs["prompt_token_ids"]
680 model_config = ctx.model_config
681 tokenizer = cached_get_tokenizer(
682 model_config.tokenizer,
@@ -713,9 +714,9 @@ def input_processor_for_qwen(ctx: InputContext,
714
715 new_prompt_token_ids = tokenizer.encode(new_prompt)
716
716 - return LLMInputs(prompt=new_prompt,
717 - prompt_token_ids=new_prompt_token_ids,
718 - multi_modal_data=multi_modal_data)
717 + return token_inputs(prompt=new_prompt,
718 + prompt_token_ids=new_prompt_token_ids,
719 + multi_modal_data=multi_modal_data)
720
721
722 def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
@@ -822,7 +823,7 @@ def dummy_data_for_qwen(
823 # The presence of a visual config indicates this is a multimodal model.
824 # If we don't have it, the model is considered an LLM for warmup purposes.
825 if not hasattr(hf_config, "visual"):
825 - seq_data = SequenceData.from_token_counts((0, seq_len))
826 + seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
827 mm_data = None
828 return seq_data, mm_data
829
vllm/model_executor/models/qwen2_vl.py
+14 -11
@@ -46,7 +46,8 @@ from vllm.attention.selector import (_Backend, backend_name_to_enum,
46 from vllm.config import CacheConfig, MultiModalConfig
47 from vllm.distributed import get_pp_group, parallel_state
48 from vllm.distributed import utils as dist_utils
49 -from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
49 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
50 + token_inputs)
51 from vllm.logger import init_logger
52 from vllm.model_executor import SamplingMetadata
53 from vllm.model_executor.layers.activation import QuickGELU
@@ -716,7 +717,7 @@ def dummy_data_for_qwen2_vl(
717
718 hf_config = ctx.get_hf_config(Qwen2VLConfig)
719
719 - dummy_seqdata = SequenceData.from_token_counts(
720 + dummy_seqdata = SequenceData.from_prompt_token_counts(
721 (hf_config.vision_start_token_id, 1),
722 (hf_config.image_token_id, max_llm_image_tokens),
723 (hf_config.vision_end_token_id, 1),
@@ -799,11 +800,13 @@ def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
800 return prompt_token_ids_with_data
801
802
802 -def input_processor_for_qwen2_vl(ctx: InputContext,
803 - llm_inputs: LLMInputs) -> LLMInputs:
804 - multi_modal_data = llm_inputs.get("multi_modal_data", None)
803 +def input_processor_for_qwen2_vl(
804 + ctx: InputContext,
805 + inputs: DecoderOnlyInputs,
806 +) -> DecoderOnlyInputs:
807 + multi_modal_data = inputs.get("multi_modal_data", None)
808 if multi_modal_data is None:
806 - return llm_inputs
809 + return inputs
810
811 image_inputs = multi_modal_data.get("image", None)
812 video_inputs = multi_modal_data.get("video", None)
@@ -817,7 +820,7 @@ def input_processor_for_qwen2_vl(ctx: InputContext,
820 # `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2VLProcessor`.
821 #
822 # The following code is equivalent to:
820 - # prompt = llm_inputs["prompt"]
823 + # prompt = inputs["prompt"]
824 # inputs = processor(text=[prompt],
825 # images=image_inputs,
826 # videos=video_inputs,
@@ -825,9 +828,9 @@ def input_processor_for_qwen2_vl(ctx: InputContext,
828 # return_tensors="pt")
829 # prompt_token_ids = inputs["input_ids"][0].tolist()
830
828 - prompt_token_ids = llm_inputs.get("prompt_token_ids", None)
831 + prompt_token_ids = inputs.get("prompt_token_ids", None)
832 if prompt_token_ids is None:
830 - prompt = llm_inputs["prompt"]
833 + prompt = inputs["prompt"]
834 prompt_token_ids = processor.tokenizer(
835 prompt,
836 padding=True,
@@ -868,9 +871,9 @@ def input_processor_for_qwen2_vl(ctx: InputContext,
871 image_processor,
872 prompt_token_ids)
873
871 - return LLMInputs(
874 + return token_inputs(
875 prompt_token_ids=prompt_token_ids,
873 - prompt=llm_inputs["prompt"],
876 + prompt=inputs["prompt"],
877 multi_modal_data=multi_modal_data,
878 )
879
vllm/model_executor/models/siglip.py
+8 -8
@@ -13,7 +13,7 @@ from transformers.models.siglip.modeling_siglip import SiglipSdpaAttention
13
14 from vllm.config import ModelConfig
15 from vllm.distributed import divide, get_tensor_model_parallel_world_size
16 -from vllm.inputs import LLMInputs
16 +from vllm.inputs import DecoderOnlyInputs, token_inputs
17 from vllm.model_executor.layers.activation import get_act_fn
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 QKVParallelLinear,
@@ -67,7 +67,7 @@ def dummy_seq_data_for_siglip(
67 else:
68 image_feature_size = image_feature_size_override
69
70 - return SequenceData.from_token_counts(
70 + return SequenceData.from_prompt_token_counts(
71 (image_token_id, image_feature_size * num_images),
72 (0, seq_len - image_feature_size * num_images),
73 )
@@ -111,14 +111,14 @@ def dummy_video_for_siglip(
111 def input_processor_for_siglip(
112 model_config: ModelConfig,
113 hf_config: SiglipVisionConfig,
114 - llm_inputs: LLMInputs,
114 + inputs: DecoderOnlyInputs,
115 *,
116 image_token_id: int,
117 image_feature_size_override: Optional[Union[int, List[int]]] = None,
118 ):
119 - multi_modal_data = llm_inputs.get("multi_modal_data")
119 + multi_modal_data = inputs.get("multi_modal_data")
120 if multi_modal_data is None or "image" not in multi_modal_data:
121 - return llm_inputs
121 + return inputs
122
123 tokenizer = cached_get_tokenizer(model_config.tokenizer)
124
@@ -135,14 +135,14 @@ def input_processor_for_siglip(
135
136 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
137 tokenizer,
138 - llm_inputs.get("prompt"),
139 - llm_inputs["prompt_token_ids"],
138 + inputs.get("prompt"),
139 + inputs["prompt_token_ids"],
140 placeholder_token_id=image_token_id,
141 repeat_count=image_feature_size,
142 )
143
144 # NOTE: Create a defensive copy of the original inputs
145 - return LLMInputs(
145 + return token_inputs(
146 prompt_token_ids=new_token_ids,
147 prompt=new_prompt,
148 multi_modal_data=multi_modal_data,
vllm/model_executor/models/ultravox.py
+9 -9
@@ -18,7 +18,7 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
18 from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, MultiModalConfig
20 from vllm.inputs import INPUT_REGISTRY
21 -from vllm.inputs.data import LLMInputs
21 +from vllm.inputs.data import DecoderOnlyInputs, token_inputs
22 from vllm.inputs.registry import InputContext
23 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
24 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -156,10 +156,10 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
156 return MultiModalInputs({"audio_features": audio_features})
157
158
159 -def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
160 - multi_modal_data = llm_inputs.get("multi_modal_data")
159 +def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
160 + multi_modal_data = inputs.get("multi_modal_data")
161 if multi_modal_data is None or "audio" not in multi_modal_data:
162 - return llm_inputs
162 + return inputs
163
164 feature_extractor = whisper_feature_extractor(ctx)
165 audios = multi_modal_data["audio"]
@@ -196,16 +196,16 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
196
197 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
198 tokenizer,
199 - llm_inputs.get("prompt"),
200 - llm_inputs["prompt_token_ids"],
199 + inputs.get("prompt"),
200 + inputs["prompt_token_ids"],
201 placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN,
202 repeat_count=audio_token_counts,
203 )
204
205 # NOTE: Create a defensive copy of the original inputs
206 - return LLMInputs(prompt_token_ids=new_token_ids,
207 - prompt=new_prompt,
208 - multi_modal_data=multi_modal_data)
206 + return token_inputs(prompt_token_ids=new_token_ids,
207 + prompt=new_prompt,
208 + multi_modal_data=multi_modal_data)
209
210
211 class StackAudioFrames(nn.Module):
vllm/sequence.py
+47 -27
@@ -13,8 +13,7 @@ from typing import Set, Tuple, Union, cast
13 import msgspec
14 import torch
15
16 -from vllm.inputs import EncoderDecoderLLMInputs, LLMInputs
17 -from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
16 +from vllm.inputs.parse import is_encoder_decoder_inputs
17 from vllm.lora.request import LoRARequest
18 from vllm.pooling_params import PoolingParams
19 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -22,6 +21,7 @@ from vllm.sampling_params import SamplingParams
21 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
22
23 if TYPE_CHECKING:
24 + from vllm.inputs import SingletonInputs
25 from vllm.multimodal.base import MultiModalDataDict
26
27 VLLM_TOKEN_ID_ARRAY_TYPE = "l"
@@ -29,6 +29,11 @@ VLLM_TOKEN_ID_ARRAY_TYPE = "l"
29 VLLM_INVALID_TOKEN_ID = -1
30
31
32 +def array_full(token_id: int, count: int):
33 + """:class:`array` equivalent of :func:`numpy.full`."""
34 + return array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
35 +
36 +
37 # We use dataclass for now because it is used for
38 # openai server output, and msgspec is not serializable.
39 # TODO(sang): Fix it.
@@ -173,22 +178,34 @@ class SequenceData(msgspec.Struct,
178 _mrope_position_delta: Optional[int] = None
179
180 @staticmethod
176 - def from_token_counts(*token_counts: Tuple[int, int]) -> "SequenceData":
181 + def from_prompt_token_counts(
182 + *token_counts: Tuple[int, int]) -> "SequenceData":
183 + """
184 + Construct a :class:`SequenceData` instance by concatenating
185 + prompt token sequences.
186 +
187 + Each tuple represents one token sequence, expressed in the form
188 + :code:`(token_id, count)`.
189 + """
190 if len(token_counts) == 0:
191 return SequenceData.from_seqs([])
192
180 - arrs = [
181 - array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
182 - for token_id, count in token_counts
183 - ]
193 + prompt_token_ids_arr = reduce(
194 + array.__iadd__,
195 + (array_full(token_id, count) for token_id, count in token_counts),
196 + )
197
185 - return SequenceData(reduce(array.__add__, arrs))
198 + return SequenceData(prompt_token_ids_arr)
199
200 @staticmethod
201 def from_seqs(
202 prompt_token_ids: GenericSequence[int],
203 output_token_ids: Optional[GenericSequence[int]] = None,
204 ) -> "SequenceData":
205 + """
206 + Construct a :class:`SequenceData` instance from prompt and output
207 + token sequences.
208 + """
209 prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
210 prompt_token_ids)
211
@@ -362,14 +379,14 @@ class SequenceData(msgspec.Struct,
379 class Sequence:
380 """Stores the data, status, and block information of a sequence.
381
365 - The sequence is constructed from the LLMInputs instance passed
366 - in through the `inputs` constructor argument.
382 + The sequence is constructed from the :code:`SingletonInputs` instance
383 + passed in through the :code:`inputs` constructor argument.
384
368 - For encoder/decoder models, LLMInputs encapsulates both a
385 + For encoder/decoder models, SingletonInputs encapsulates both a
386 decoder and encoder prompt, creating an ambiguity about which
387 prompt to construct the sequence from. The `from_decoder_prompt`
388 constructor argument signals whether to construct the Sequence
372 - from the LLMInputs decoder prompt, or encoder prompt.
389 + from the SingletonInputs decoder prompt, or encoder prompt.
390
391 Args:
392 seq_id: The ID of the sequence.
@@ -379,16 +396,16 @@ class Sequence:
396 eos_token_id: The end-of-sequence (EOS) token id recognized by this LLM.
397 lora_request: LoRA request.
398 prompt_adapter_request: Prompt Adapter request.
382 - from_decoder_prompt: Construct Sequence from LLMInputs decoder prompt
383 - (True) or encoder prompt (False.) Must be True
384 - for decoder-only model.
399 + from_decoder_prompt: Construct Sequence from SingletonInputs decoder
400 + prompt (True) or encoder prompt (False.) Must be
401 + True for decoder-only model.
402
403 """
404
405 def __init__(
406 self,
407 seq_id: int,
391 - inputs: "LLMInputs",
408 + inputs: "SingletonInputs",
409 block_size: int,
410 eos_token_id: Optional[int] = None,
411 lora_request: Optional[LoRARequest] = None,
@@ -404,19 +421,19 @@ class Sequence:
421 self.from_decoder_prompt = from_decoder_prompt
422
423 # For decoder-only models, a Sequence is constructed
407 - # from an LLMInputs instance (the `inputs` arg.)
424 + # from an DecoderOnlyInputs instance (the `inputs` arg.)
425 #
426 # For encoder/decoder models the same `inputs`
427 # instance could be utilized to construct either an
428 # encoder sequence or a decoder sequence, because
412 - # `LLMInputs` has both decoder- and encoder-oriented
429 + # `DecoderOnlyInputs` has both decoder- and encoder-oriented
430 # member variables (i.e. it encapsulates both an encoder
431 # and a decoder prompt.) The decision of which type of sequence
432 # to generate is determined by the `from_decoder_prompt` argument.
433 #
434 # When constructing a encoder sequence
435 # (`from_decoder_prompt` False) it matters that
419 - # the `LLMInputs` instance stored in `inputs` is valid
436 + # the `DecoderOnlyInputs` instance stored in `inputs` is valid
437 # in the sense that its encoder-related member variables are
438 # populated; below, an exception is raised if this is
439 # not the case.
@@ -424,8 +441,7 @@ class Sequence:
441 # When constructing a decoder sequence (`from_decoder_prompt` True)
442 # it does not matter whether `inputs` has its encoder-related
443 # member variables populated.
427 - if not (from_decoder_prompt
428 - or is_valid_encoder_decoder_llm_inputs(inputs)):
444 + if not (from_decoder_prompt or is_encoder_decoder_inputs(inputs)):
445 raise ValueError("Cannot extract encoder input prompt from "
446 f"invalid input {inputs}; did you forget the "
447 "encoder input prompt fields?")
@@ -471,15 +487,19 @@ class Sequence:
487
488 @property
489 def multi_modal_data(self) -> "MultiModalDataDict":
474 - if self.inputs.get("multi_modal_data") and self.inputs.get(
475 - "encoder_multi_modal_data"):
490 + inputs = self.inputs
491 +
492 + if (inputs.get("multi_modal_data")
493 + and inputs.get("encoder_multi_modal_data")):
494 raise ValueError(
495 "Multi-modal data in both encoder and decoder is not supported."
496 )
479 - inputs = self.inputs
480 - return self.inputs.get("multi_modal_data") or (cast(
481 - EncoderDecoderLLMInputs,
482 - inputs).get("encoder_multi_modal_data")) or {}
497 +
498 + return cast(
499 + "MultiModalDataDict",
500 + (inputs.get("multi_modal_data")
501 + or inputs.get("encoder_multi_modal_data") or {}),
502 + )
503
504 @property
505 def mm_processor_kwargs(self) -> Dict[str, Any]: