A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core] Rename input data types (#8688)
Cyrus Leung committed
Oct 16, 2024 at 18:49 UTC
cee711fdbb88d7c6506a9039d74cb2911c516f94
32 files changed
+438
-340
docs/source/dev/input_processing/model_inputs_index.rst
+1
-1
@@ -25,7 +25,7 @@ Module Contents
25
LLM Engine Inputs
26
-----------------
27
28
-.. autoclass:: vllm.inputs.LLMInputs
28
+.. autoclass:: vllm.inputs.DecoderOnlyInputs
29
:members:
30
:show-inheritance:
31
tests/models/decoder_only/vision_language/test_phi3v.py
+13
-15
@@ -1,12 +1,12 @@
1
import os
2
import re
3
-from typing import Callable, List, Optional, Tuple, Type
3
+from typing import List, Optional, Tuple, Type
4
5
import pytest
6
import torch
7
from transformers import AutoImageProcessor, AutoTokenizer
8
9
-from vllm.inputs import InputContext, LLMInputs
9
+from vllm.inputs import InputContext, token_inputs
10
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
11
from vllm.multimodal import MultiModalRegistry
12
from vllm.multimodal.utils import rescale_image_size
@@ -311,7 +311,7 @@ def test_input_mapper_override(model: str, image_assets: _ImageAssets,
311
(4, 781),
312
(16, 2653),
313
])
314
-def test_max_tokens_override(get_max_phi3v_image_tokens: Callable, model: str,
314
+def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
315
num_crops: int, expected_max_tokens: int):
316
"""Ensure get_max_phi3v_image_tokens handles num_crops properly."""
317
# NOTE: mm_processor_kwargs on the context in this test is unused, since
@@ -343,8 +343,8 @@ def test_max_tokens_override(get_max_phi3v_image_tokens: Callable, model: str,
343
(16, 2653, 1),
344
(16, 2653, 2),
345
])
346
-def test_dummy_data_override(dummy_data_for_phi3v: Callable, model: str,
347
- num_crops: int, toks_per_img: int, num_imgs: int):
346
+def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
347
+ toks_per_img: int, num_imgs: int):
348
"""Ensure dummy_data_for_phi3v handles num_crops properly."""
349
# Same as the previous test - don't initialize mm_processor_kwargs
350
# in this test and assume that the kwargs will be correctly expanded by
@@ -374,7 +374,7 @@ def test_dummy_data_override(dummy_data_for_phi3v: Callable, model: str,
374
(16, 1921, 1),
375
(16, 1921, 2),
376
])
377
-def test_input_processor_override(input_processor_for_phi3v: Callable,
377
+def test_input_processor_override(input_processor_for_phi3v,
378
image_assets: _ImageAssets, model: str,
379
num_crops: int, expected_toks_per_img: int,
380
num_imgs: int):
@@ -393,16 +393,14 @@ def test_input_processor_override(input_processor_for_phi3v: Callable,
393
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
394
images = [image_assets[0].pil_image] * num_imgs
395
396
- llm_inputs = LLMInputs(prompt_token_ids=tokenizer.encode(prompt),
397
- prompt=prompt,
398
- multi_modal_data={"image": images})
396
+ inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
397
+ prompt=prompt,
398
+ multi_modal_data={"image": images})
399
400
- proc_llm_inputs = input_processor_for_phi3v(
401
- ctx=ctx,
402
- llm_inputs=llm_inputs,
403
- num_crops=num_crops,
404
- )
400
+ processed_inputs = input_processor_for_phi3v(ctx,
401
+ inputs,
402
+ num_crops=num_crops)
403
404
# Ensure we have the right number of placeholders per num_crops size
407
- img_tok_count = proc_llm_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
405
+ img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
406
assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/decoder_only/vision_language/test_qwen.py
+6
-6
@@ -5,7 +5,7 @@ import pytest
5
import torch
6
from PIL.Image import Image
7
8
-from vllm.inputs import InputContext, LLMInputs
8
+from vllm.inputs import InputContext, token_inputs
9
from vllm.multimodal.base import MultiModalInputs
10
from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
11
@@ -71,12 +71,12 @@ def test_input_processor_valid_mm_data(input_processor_for_qwen,
71
"""Happy cases for image inputs to Qwen's multimodal input processor."""
72
prompt = "".join(
73
[f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
74
- inputs = LLMInputs(
74
+ inputs = token_inputs(
75
prompt=prompt,
76
# When processing multimodal data for a multimodal model, the qwen
77
# input processor will overwrite the provided prompt_token_ids with
78
# the image prompts
79
- prompt_token_ids=None,
79
+ prompt_token_ids=[],
80
multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
81
)
82
proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
@@ -134,9 +134,9 @@ def test_input_processor_invalid_mm_data(input_processor_for_qwen,
134
trust_remote_code=True)
135
prompt = "Picture 1: <img></img>\n"
136
prompt_token_ids = tokenizer.encode(prompt)
137
- inputs = LLMInputs(prompt=prompt,
138
- prompt_token_ids=prompt_token_ids,
139
- multi_modal_data=mm_data)
137
+ inputs = token_inputs(prompt=prompt,
138
+ prompt_token_ids=prompt_token_ids,
139
+ multi_modal_data=mm_data)
140
# Should fail since we have too many or too few dimensions for embeddings
141
with pytest.raises(ValueError):
142
input_processor_for_qwen(qwen_vl_context, inputs)
tests/multimodal/test_processor_kwargs.py
+9
-9
@@ -5,7 +5,7 @@ from unittest.mock import patch
5
import pytest
6
import torch
7
8
-from vllm.inputs import InputContext, LLMInputs
8
+from vllm.inputs import DecoderOnlyInputs, InputContext, token_inputs
9
from vllm.inputs.registry import InputRegistry
10
from vllm.multimodal import MultiModalRegistry
11
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
@@ -31,7 +31,7 @@ def use_processor_mock():
31
"""Patches the internal model input processor with an override callable."""
32
33
def custom_processor(ctx: InputContext,
34
- llm_inputs: LLMInputs,
34
+ inputs: DecoderOnlyInputs,
35
*,
36
num_crops=DEFAULT_NUM_CROPS):
37
# For testing purposes, we don't worry about the llm inputs / return
@@ -84,7 +84,7 @@ def test_default_processor_is_a_noop():
84
dummy_registry = InputRegistry()
85
ctx = build_model_context(DUMMY_MODEL_ID)
86
processor = dummy_registry.create_input_processor(ctx.model_config)
87
- proc_inputs = LLMInputs(prompt_token_ids=[], prompt="")
87
+ proc_inputs = token_inputs(prompt_token_ids=[], prompt="")
88
proc_outputs = processor(inputs=proc_inputs)
89
assert proc_inputs is proc_outputs
90
@@ -125,9 +125,9 @@ def test_input_processor_kwargs(use_processor_mock, init_num_crops,
125
ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
126
processor = dummy_registry.create_input_processor(ctx.model_config)
127
num_crops_val = processor(
128
- LLMInputs(prompt_token_ids=[],
129
- prompt="",
130
- mm_processor_kwargs=inference_kwargs))
128
+ token_inputs(prompt_token_ids=[],
129
+ prompt="",
130
+ mm_processor_kwargs=inference_kwargs))
131
assert num_crops_val == expected_seq_count
132
133
@@ -154,9 +154,9 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
154
processor = dummy_registry.create_input_processor(ctx.model_config)
155
# Should filter out the inference time kwargs
156
num_crops_val = processor(
157
- LLMInputs(prompt_token_ids=[],
158
- prompt="",
159
- mm_processor_kwargs=mm_processor_kwargs))
157
+ token_inputs(prompt_token_ids=[],
158
+ prompt="",
159
+ mm_processor_kwargs=mm_processor_kwargs))
160
assert num_crops_val == DEFAULT_NUM_CROPS
161
162
vllm/engine/llm_engine.py
+5
-5
@@ -29,8 +29,8 @@ from vllm.entrypoints.openai.logits_processors import get_logits_processors
29
from vllm.executor.executor_base import ExecutorBase
30
from vllm.executor.gpu_executor import GPUExecutor
31
from vllm.executor.ray_utils import initialize_ray_cluster
32
-from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
33
- InputRegistry, LLMInputs, PromptType)
32
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
33
+ EncoderDecoderInputs, InputRegistry, PromptType)
34
from vllm.inputs.preprocess import InputPreprocessor
35
from vllm.logger import init_logger
36
from vllm.lora.request import LoRARequest
@@ -635,7 +635,7 @@ class LLMEngine:
635
def _add_processed_request(
636
self,
637
request_id: str,
638
- processed_inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
638
+ processed_inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
639
params: Union[SamplingParams, PoolingParams],
640
arrival_time: float,
641
lora_request: Optional[LoRARequest],
@@ -1855,8 +1855,8 @@ class LLMEngine:
1855
def is_embedding_model(self):
1856
return self.model_config.is_embedding_model
1857
1858
- def _validate_model_inputs(self, inputs: Union[LLMInputs,
1859
- EncoderDecoderLLMInputs]):
1858
+ def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1859
+ EncoderDecoderInputs]):
1860
if self.model_config.is_multimodal_model:
1861
# For encoder-decoder multimodal models, the max_prompt_len
1862
# restricts the decoder prompt length
vllm/inputs/__init__.py
+29
-8
@@ -1,7 +1,8 @@
1
-from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2
- LLMInputs, PromptType, SingletonPrompt, TextPrompt,
3
- TokensPrompt, build_explicit_enc_dec_prompt,
4
- to_enc_dec_tuple_list, zip_enc_dec_prompts)
1
+from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
2
+ ExplicitEncoderDecoderPrompt, PromptType, SingletonInputs,
3
+ SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
4
+ build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
5
+ token_inputs, zip_enc_dec_prompts)
6
from .registry import InputContext, InputRegistry
7
8
INPUT_REGISTRY = InputRegistry()
@@ -19,8 +20,11 @@ __all__ = [
20
"PromptType",
21
"SingletonPrompt",
22
"ExplicitEncoderDecoderPrompt",
22
- "LLMInputs",
23
- "EncoderDecoderLLMInputs",
23
+ "TokenInputs",
24
+ "token_inputs",
25
+ "SingletonInputs",
26
+ "DecoderOnlyInputs",
27
+ "EncoderDecoderInputs",
28
"build_explicit_enc_dec_prompt",
29
"to_enc_dec_tuple_list",
30
"zip_enc_dec_prompts",
@@ -31,9 +35,9 @@ __all__ = [
35
36
37
def __getattr__(name: str):
34
- if name == "PromptInput":
35
- import warnings
38
+ import warnings
39
40
+ if name == "PromptInput":
41
msg = ("PromptInput has been renamed to PromptType. "
42
"The original name will be removed in an upcoming version.")
43
@@ -41,4 +45,21 @@ def __getattr__(name: str):
45
46
return PromptType
47
48
+ if name == "LLMInputs":
49
+ msg = ("LLMInputs has been renamed to DecoderOnlyInputs. "
50
+ "The original name will be removed in an upcoming version.")
51
+
52
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
53
+
54
+ return DecoderOnlyInputs
55
+
56
+ if name == "EncoderDecoderLLMInputs":
57
+ msg = (
58
+ "EncoderDecoderLLMInputs has been renamed to EncoderDecoderInputs. "
59
+ "The original name will be removed in an upcoming version.")
60
+
61
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
62
+
63
+ return EncoderDecoderInputs
64
+
65
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/data.py
+62
-16
@@ -1,5 +1,5 @@
1
from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
2
- Optional, Tuple, Union)
2
+ Optional, Tuple, Union, cast)
3
4
from typing_extensions import NotRequired, TypedDict, TypeVar
5
@@ -51,7 +51,7 @@ class TokensPrompt(TypedDict):
51
52
SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
53
"""
54
-Set of possible schemas for a single LLM input:
54
+Set of possible schemas for a single prompt:
55
56
- A text prompt (:class:`str` or :class:`TextPrompt`)
57
- A tokenized prompt (:class:`TokensPrompt`)
@@ -120,13 +120,8 @@ both decoder-only and encoder/decoder input types:
120
"""
121
122
123
-class LLMInputs(TypedDict):
124
- """
125
- The inputs in :class:`~vllm.LLMEngine` before they are
126
- passed to the model executor.
127
-
128
- This specifies the data required for decoder-only models.
129
- """
123
+class TokenInputs(TypedDict):
124
+ """Represents token-based inputs."""
125
prompt_token_ids: List[int]
126
"""The token IDs of the prompt."""
127
@@ -150,7 +145,40 @@ class LLMInputs(TypedDict):
145
"""
146
147
153
-class EncoderDecoderLLMInputs(LLMInputs):
148
+def token_inputs(
149
+ prompt_token_ids: List[int],
150
+ prompt: Optional[str] = None,
151
+ multi_modal_data: Optional["MultiModalDataDict"] = None,
152
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
153
+) -> TokenInputs:
154
+ """Construct :class:`TokenInputs` from optional values."""
155
+ inputs = TokenInputs(prompt_token_ids=prompt_token_ids)
156
+
157
+ if prompt is not None:
158
+ inputs["prompt"] = prompt
159
+ if multi_modal_data is not None:
160
+ inputs["multi_modal_data"] = multi_modal_data
161
+ if mm_processor_kwargs is not None:
162
+ inputs["mm_processor_kwargs"] = mm_processor_kwargs
163
+
164
+ return inputs
165
+
166
+
167
+SingletonInputs = TokenInputs
168
+"""
169
+A processed :class:`SingletonPrompt` which can be passed to
170
+:class:`vllm.sequence.Sequence`.
171
+"""
172
+
173
+DecoderOnlyInputs = TokenInputs
174
+"""
175
+The inputs in :class:`~vllm.LLMEngine` before they are
176
+passed to the model executor.
177
+This specifies the data required for decoder-only models.
178
+"""
179
+
180
+
181
+class EncoderDecoderInputs(TokenInputs):
182
"""
183
The inputs in :class:`~vllm.LLMEngine` before they are
184
passed to the model executor.
@@ -204,11 +232,12 @@ def zip_enc_dec_prompts(
232
be zipped with the encoder/decoder prompts.
233
"""
234
if mm_processor_kwargs is None:
207
- mm_processor_kwargs = {}
208
- if isinstance(mm_processor_kwargs, Dict):
235
+ mm_processor_kwargs = cast(Dict[str, Any], {})
236
+ if isinstance(mm_processor_kwargs, dict):
237
return [
210
- build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
211
- mm_processor_kwargs)
238
+ build_explicit_enc_dec_prompt(
239
+ encoder_prompt, decoder_prompt,
240
+ cast(Dict[str, Any], mm_processor_kwargs))
241
for (encoder_prompt,
242
decoder_prompt) in zip(enc_prompts, dec_prompts)
243
]
@@ -229,9 +258,9 @@ def to_enc_dec_tuple_list(
258
259
260
def __getattr__(name: str):
232
- if name == "PromptInput":
233
- import warnings
261
+ import warnings
262
263
+ if name == "PromptInput":
264
msg = ("PromptInput has been renamed to PromptType. "
265
"The original name will be removed in an upcoming version.")
266
@@ -239,4 +268,21 @@ def __getattr__(name: str):
268
269
return PromptType
270
271
+ if name == "LLMInputs":
272
+ msg = ("LLMInputs has been renamed to DecoderOnlyInputs. "
273
+ "The original name will be removed in an upcoming version.")
274
+
275
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
276
+
277
+ return DecoderOnlyInputs
278
+
279
+ if name == "EncoderDecoderLLMInputs":
280
+ msg = (
281
+ "EncoderDecoderLLMInputs has been renamed to EncoderDecoderInputs. "
282
+ "The original name will be removed in an upcoming version.")
283
+
284
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
285
+
286
+ return EncoderDecoderInputs
287
+
288
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/parse.py
+6
-6
@@ -4,9 +4,9 @@ from typing_extensions import TypeIs
4
5
from vllm.utils import is_list_of
6
7
-from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8
- LLMInputs, PromptType, SingletonPrompt, TextPrompt,
9
- TokensPrompt)
7
+from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
8
+ ExplicitEncoderDecoderPrompt, PromptType, SingletonPrompt,
9
+ TextPrompt, TokensPrompt)
10
11
12
class ParsedText(TypedDict):
@@ -100,7 +100,7 @@ def is_explicit_encoder_decoder_prompt(
100
return isinstance(prompt, dict) and "encoder_prompt" in prompt
101
102
103
-def is_valid_encoder_decoder_llm_inputs(
104
- inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
105
-) -> TypeIs[EncoderDecoderLLMInputs]:
103
+def is_encoder_decoder_inputs(
104
+ inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
105
+) -> TypeIs[EncoderDecoderInputs]:
106
return "encoder_prompt_token_ids" in inputs
vllm/inputs/preprocess.py
+18
-18
@@ -10,7 +10,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
10
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
from vllm.utils import print_warning_once
12
13
-from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
13
+from .data import (DecoderOnlyInputs, EncoderDecoderInputs, PromptType,
14
SingletonPrompt)
15
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
16
@@ -306,7 +306,7 @@ class InputPreprocessor:
306
encoder_comps: PromptComponents,
307
decoder_comps: DecoderPromptComponents,
308
mm_processor_kwargs: Dict[str, Any],
309
- ) -> EncoderDecoderLLMInputs:
309
+ ) -> EncoderDecoderInputs:
310
encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311
decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312
@@ -324,7 +324,7 @@ class InputPreprocessor:
324
decoder_prompt_ids,
325
force_bos=(encoder_mm_data is None and decoder_mm_data is None)))
326
327
- return EncoderDecoderLLMInputs(
327
+ return EncoderDecoderInputs(
328
prompt_token_ids=decoder_prompt_ids,
329
prompt=decoder_prompt,
330
multi_modal_data=decoder_mm_data,
@@ -338,11 +338,11 @@ class InputPreprocessor:
338
self,
339
prompt: PromptType,
340
request_id: str,
341
- ) -> EncoderDecoderLLMInputs:
341
+ ) -> EncoderDecoderInputs:
342
'''
343
For encoder/decoder models only:
344
Process an input prompt into an
345
- :class:`EncoderDecoderLLMInputs` instance.
345
+ :class:`EncoderDecoderInputs` instance.
346
347
There are two types of input prompts:
348
singleton prompts which carry only the
@@ -369,7 +369,7 @@ class InputPreprocessor:
369
370
Returns:
371
372
- * :class:`EncoderDecoderLLMInputs` instance
372
+ * :class:`EncoderDecoderInputs` instance
373
'''
374
375
encoder_comps: PromptComponents
@@ -411,7 +411,7 @@ class InputPreprocessor:
411
self,
412
prompt: PromptType,
413
request_id: str,
414
- ) -> EncoderDecoderLLMInputs:
414
+ ) -> EncoderDecoderInputs:
415
"""Async version of :meth:`_process_encoder_decoder_prompt`."""
416
encoder_comps: PromptComponents
417
decoder_comps: DecoderPromptComponents
@@ -455,17 +455,17 @@ class InputPreprocessor:
455
self,
456
prompt_comps: PromptComponents,
457
prompt_adapter_request: Optional[PromptAdapterRequest],
458
- ) -> LLMInputs:
458
+ ) -> DecoderOnlyInputs:
459
(prompt, prompt_token_ids, multi_modal_data,
460
mm_processor_kwargs) = prompt_comps
461
462
prompt_token_ids = self._apply_prompt_adapter(
463
prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
464
465
- return LLMInputs(prompt_token_ids=prompt_token_ids,
466
- prompt=prompt,
467
- multi_modal_data=multi_modal_data,
468
- mm_processor_kwargs=mm_processor_kwargs)
465
+ return DecoderOnlyInputs(prompt_token_ids=prompt_token_ids,
466
+ prompt=prompt,
467
+ multi_modal_data=multi_modal_data,
468
+ mm_processor_kwargs=mm_processor_kwargs)
469
470
def _process_decoder_only_prompt(
471
self,
@@ -473,10 +473,10 @@ class InputPreprocessor:
473
request_id: str,
474
lora_request: Optional[LoRARequest] = None,
475
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
476
- ) -> LLMInputs:
476
+ ) -> DecoderOnlyInputs:
477
'''
478
For decoder-only models:
479
- Process an input prompt into an :class:`LLMInputs` instance.
479
+ Process an input prompt into an :class:`DecoderOnlyInputs` instance.
480
481
Arguments:
482
@@ -487,7 +487,7 @@ class InputPreprocessor:
487
488
Returns:
489
490
- * :class:`LLMInputs` instance
490
+ * :class:`DecoderOnlyInputs` instance
491
'''
492
493
prompt_comps = self._extract_prompt_components(
@@ -507,7 +507,7 @@ class InputPreprocessor:
507
request_id: str,
508
lora_request: Optional[LoRARequest] = None,
509
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
510
- ) -> LLMInputs:
510
+ ) -> DecoderOnlyInputs:
511
"""Async version of :meth:`_process_decoder_only_prompt`."""
512
prompt_comps = await self._extract_prompt_components_async(
513
prompt,
@@ -526,7 +526,7 @@ class InputPreprocessor:
526
request_id: str,
527
lora_request: Optional[LoRARequest] = None,
528
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
529
- ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
529
+ ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
530
"""Preprocess the input prompt."""
531
if self.is_encoder_decoder_model():
532
# Encoder-decoder model requires special mapping of
@@ -554,7 +554,7 @@ class InputPreprocessor:
554
request_id: str,
555
lora_request: Optional[LoRARequest] = None,
556
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
557
- ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
557
+ ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
558
"""Async version of :meth:`preprocess`."""
559
if self.is_encoder_decoder_model():
560
# Encoder-decoder model requires special mapping of
vllm/inputs/registry.py
+9
-6
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
12
from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
13
resolve_mm_processor_kwargs)
14
15
-from .data import LLMInputs
15
+from .data import DecoderOnlyInputs
16
17
if TYPE_CHECKING:
18
from vllm.config import ModelConfig
@@ -100,7 +100,7 @@ class _MultiModalCounts(UserDict):
100
raise KeyError(msg) from exc
101
102
103
-InputProcessor = Callable[[InputContext, LLMInputs], LLMInputs]
103
+InputProcessor = Callable[[InputContext, DecoderOnlyInputs], DecoderOnlyInputs]
104
"""Preprocess the inputs to the model."""
105
106
@@ -134,7 +134,7 @@ class InputRegistry:
134
# Avoid circular import
135
from vllm.sequence import SequenceData
136
137
- dummy_seq_data = SequenceData.from_token_counts((0, seq_len))
137
+ dummy_seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
138
dummy_multi_modal_data = None
139
140
return dummy_seq_data, dummy_multi_modal_data
@@ -245,8 +245,11 @@ class InputRegistry:
245
246
return seq_data, mm_data
247
248
- def _default_input_processor(self, ctx: InputContext,
249
- inputs: LLMInputs) -> LLMInputs:
248
+ def _default_input_processor(
249
+ self,
250
+ ctx: InputContext,
251
+ inputs: DecoderOnlyInputs,
252
+ ) -> DecoderOnlyInputs:
253
"""The default input processor is a no-op."""
254
return inputs
255
@@ -279,7 +282,7 @@ class InputRegistry:
282
.get(model_cls, self._default_input_processor)
283
284
def process_input(self, model_config: "ModelConfig",
282
- inputs: LLMInputs) -> LLMInputs:
285
+ inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
286
"""
287
Apply an input processor to an instance of model inputs.
288
vllm/model_executor/models/blip.py
+10
-10
@@ -10,7 +10,7 @@ from transformers.models.blip.modeling_blip import BlipAttention
10
11
from vllm.config import ModelConfig
12
from vllm.distributed import divide, get_tensor_model_parallel_world_size
13
-from vllm.inputs import LLMInputs
13
+from vllm.inputs import DecoderOnlyInputs, token_inputs
14
from vllm.model_executor.layers.activation import get_act_fn
15
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
QKVParallelLinear,
@@ -63,7 +63,7 @@ def dummy_seq_data_for_blip(
63
else:
64
image_feature_size = image_feature_size_override
65
66
- return SequenceData.from_token_counts(
66
+ return SequenceData.from_prompt_token_counts(
67
(image_token_id, image_feature_size * num_images),
68
(0, seq_len - image_feature_size * num_images),
69
)
@@ -89,14 +89,14 @@ def dummy_image_for_blip(
89
def input_processor_for_blip(
90
model_config: ModelConfig,
91
hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
92
- llm_inputs: LLMInputs,
92
+ inputs: DecoderOnlyInputs,
93
*,
94
image_token_id: int,
95
image_feature_size_override: Optional[int] = None,
96
):
97
- multi_modal_data = llm_inputs.get("multi_modal_data")
97
+ multi_modal_data = inputs.get("multi_modal_data")
98
if multi_modal_data is None or "image" not in multi_modal_data:
99
- return llm_inputs
99
+ return inputs
100
101
tokenizer = cached_get_tokenizer(model_config.tokenizer)
102
@@ -107,16 +107,16 @@ def input_processor_for_blip(
107
108
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
109
tokenizer,
110
- llm_inputs.get("prompt"),
111
- llm_inputs["prompt_token_ids"],
110
+ inputs.get("prompt"),
111
+ inputs["prompt_token_ids"],
112
placeholder_token_id=image_token_id,
113
repeat_count=image_feature_size,
114
)
115
116
# NOTE: Create a defensive copy of the original inputs
117
- return LLMInputs(prompt_token_ids=new_token_ids,
118
- prompt=new_prompt,
119
- multi_modal_data=multi_modal_data)
117
+ return token_inputs(prompt_token_ids=new_token_ids,
118
+ prompt=new_prompt,
119
+ multi_modal_data=multi_modal_data)
120
121
122
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
vllm/model_executor/models/blip2.py
+11
-10
@@ -9,7 +9,8 @@ from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
9
10
from vllm.attention import AttentionMetadata
11
from vllm.config import CacheConfig, MultiModalConfig
12
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
13
+ token_inputs)
14
from vllm.model_executor.layers.activation import get_act_fn
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -421,7 +422,7 @@ def dummy_seq_data_for_blip2(
422
else:
423
image_feature_size = image_feature_size_override
424
424
- return SequenceData.from_token_counts(
425
+ return SequenceData.from_prompt_token_counts(
426
(image_token_id, image_feature_size * num_images),
427
(0, seq_len - image_feature_size * num_images),
428
)
@@ -449,10 +450,10 @@ def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
450
raise NotImplementedError(msg)
451
452
452
-def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
453
- multi_modal_data = llm_inputs.get("multi_modal_data")
453
+def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
454
+ multi_modal_data = inputs.get("multi_modal_data")
455
if multi_modal_data is None or "image" not in multi_modal_data:
455
- return llm_inputs
456
+ return inputs
457
458
hf_config = ctx.get_hf_config(Blip2Config)
459
image_feature_size = get_blip2_image_feature_size(hf_config)
@@ -460,15 +461,15 @@ def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
461
# The original model places image tokens at the front
462
# https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514
463
new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
463
- new_token_ids += llm_inputs["prompt_token_ids"]
464
+ new_token_ids += inputs["prompt_token_ids"]
465
465
- new_prompt = llm_inputs.get("prompt")
466
+ new_prompt = inputs.get("prompt")
467
if new_prompt is not None:
468
new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt
469
469
- return LLMInputs(prompt_token_ids=new_token_ids,
470
- prompt=new_prompt,
471
- multi_modal_data=multi_modal_data)
470
+ return token_inputs(prompt_token_ids=new_token_ids,
471
+ prompt=new_prompt,
472
+ multi_modal_data=multi_modal_data)
473
474
475
@MULTIMODAL_REGISTRY.register_image_input_mapper()
vllm/model_executor/models/chameleon.py
+12
-10
@@ -11,7 +11,8 @@ from transformers import ChameleonConfig, ChameleonVQVAEConfig
11
from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
13
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15
+ token_inputs)
16
from vllm.model_executor.layers.activation import SiluAndMul
17
from vllm.model_executor.layers.layernorm import RMSNorm
18
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -69,7 +70,7 @@ def dummy_seq_data_for_chameleon(
70
else:
71
image_feature_size = image_feature_size_override
72
72
- return SequenceData.from_token_counts(
73
+ return SequenceData.from_prompt_token_counts(
74
(image_token_id, image_feature_size * num_images),
75
(0, seq_len - image_feature_size * num_images),
76
)
@@ -106,7 +107,8 @@ def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
107
return seq_data, mm_data
108
109
109
-def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
110
+def input_processor_for_chameleon(ctx: InputContext,
111
+ inputs: DecoderOnlyInputs):
112
113
"""
114
Processing input prompt to insert required tokens for image placeholder.
@@ -114,16 +116,16 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
116
See https://github.com/huggingface/transformers/blob/0fdea8607d7e01eb0e38a1ebeb7feee30a22f0cf/src/transformers/models/chameleon/processing_chameleon.py#L58
117
""" # noqa
118
117
- multi_modal_data = llm_inputs.get("multi_modal_data")
119
+ multi_modal_data = inputs.get("multi_modal_data")
120
if multi_modal_data is None or "image" not in multi_modal_data:
119
- return llm_inputs
121
+ return inputs
122
123
model_config = ctx.model_config
124
tokenizer = cached_get_tokenizer(model_config.tokenizer)
125
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
126
tokenizer,
125
- llm_inputs.get("prompt"),
126
- llm_inputs["prompt_token_ids"],
127
+ inputs.get("prompt"),
128
+ inputs["prompt_token_ids"],
129
placeholder_token_id=CHAMELEON_IMAGE_TOKEN_ID,
130
repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
131
pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
@@ -137,9 +139,9 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
139
new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
140
141
# NOTE: Create a defensive copy of the original inputs
140
- return LLMInputs(prompt_token_ids=new_token_ids,
141
- prompt=new_prompt,
142
- multi_modal_data=multi_modal_data)
142
+ return token_inputs(prompt_token_ids=new_token_ids,
143
+ prompt=new_prompt,
144
+ multi_modal_data=multi_modal_data)
145
146
147
class ChameleonLayerNorm(nn.LayerNorm):
vllm/model_executor/models/chatglm.py
+11
-11
@@ -14,7 +14,7 @@ from torch.nn import LayerNorm
14
from vllm.attention import Attention, AttentionMetadata
15
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16
from vllm.distributed import get_tensor_model_parallel_world_size
17
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
17
+from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
18
from vllm.logger import init_logger
19
from vllm.model_executor.layers.activation import SiluAndMul
20
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -149,20 +149,20 @@ def find_all_positions(input_ids: List[int], target: int) -> List[int]:
149
return [index for index, value in enumerate(input_ids) if value == target]
150
151
152
-def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
152
+def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
153
hf_config = ctx.get_hf_config(ChatGLMConfig)
154
vision_config = getattr(hf_config, 'vision_config', None)
155
156
if vision_config is None:
157
- return llm_inputs
157
+ return inputs
158
elif isinstance(vision_config, dict):
159
image_placeholder_length = calculate_image_placeholder(vision_config)
160
else:
161
msg = f"Unsupported vision config: {type(vision_config)}"
162
raise NotImplementedError(msg)
163
164
- input_ids = llm_inputs.get("prompt_token_ids")
165
- position_ids = llm_inputs.get("position_ids")
164
+ input_ids = inputs.get("prompt_token_ids")
165
+ position_ids = inputs.get("position_ids")
166
tokenizer = cached_get_tokenizer(
167
ctx.model_config.model,
168
trust_remote_code=ctx.model_config.trust_remote_code)
@@ -171,15 +171,15 @@ def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
171
raw_batch_data = tokenizer.apply_chat_template(
172
conversation=[{
173
"role": "user",
174
- "image": llm_inputs['multi_modal_data']["image"],
175
- "content": llm_inputs['prompt']
174
+ "image": inputs['multi_modal_data']["image"],
175
+ "content": inputs['prompt']
176
}],
177
add_generation_prompt=True,
178
tokenize=True,
179
return_tensors="pt",
180
return_dict=True).data
181
except Exception:
182
- logger.error("Failed to process content (%s)", llm_inputs['prompt'])
182
+ logger.error("Failed to process content (%s)", inputs['prompt'])
183
raise
184
input_ids = raw_batch_data['input_ids'][0].tolist()
185
@@ -214,9 +214,9 @@ def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
214
215
assert len(new_input_ids) == len(new_position_ids)
216
217
- llm_inputs["prompt_token_ids"] = new_input_ids
218
- llm_inputs["position_ids"] = new_position_ids
219
- return llm_inputs
217
+ inputs["prompt_token_ids"] = new_input_ids
218
+ inputs["position_ids"] = new_position_ids
219
+ return inputs
220
221
222
class GLMAttention(nn.Module):
vllm/model_executor/models/clip.py
+10
-10
@@ -11,7 +11,7 @@ from transformers.models.clip.modeling_clip import CLIPSdpaAttention
11
12
from vllm.config import ModelConfig
13
from vllm.distributed import divide, get_tensor_model_parallel_world_size
14
-from vllm.inputs import LLMInputs
14
+from vllm.inputs import DecoderOnlyInputs, token_inputs
15
from vllm.model_executor.layers.activation import get_act_fn
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
QKVParallelLinear,
@@ -62,7 +62,7 @@ def dummy_seq_data_for_clip(
62
else:
63
image_feature_size = image_feature_size_override
64
65
- return SequenceData.from_token_counts(
65
+ return SequenceData.from_prompt_token_counts(
66
(image_token_id, image_feature_size * num_images),
67
(0, seq_len - image_feature_size * num_images),
68
)
@@ -106,14 +106,14 @@ def dummy_video_for_clip(
106
def input_processor_for_clip(
107
model_config: ModelConfig,
108
hf_config: CLIPVisionConfig,
109
- llm_inputs: LLMInputs,
109
+ inputs: DecoderOnlyInputs,
110
*,
111
image_token_id: int,
112
image_feature_size_override: Optional[Union[int, List[int]]] = None,
113
):
114
- multi_modal_data = llm_inputs.get("multi_modal_data")
114
+ multi_modal_data = inputs.get("multi_modal_data")
115
if multi_modal_data is None or "image" not in multi_modal_data:
116
- return llm_inputs
116
+ return inputs
117
118
tokenizer = cached_get_tokenizer(model_config.tokenizer)
119
@@ -130,16 +130,16 @@ def input_processor_for_clip(
130
131
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
132
tokenizer,
133
- llm_inputs.get("prompt"),
134
- llm_inputs["prompt_token_ids"],
133
+ inputs.get("prompt"),
134
+ inputs["prompt_token_ids"],
135
placeholder_token_id=image_token_id,
136
repeat_count=image_feature_size,
137
)
138
139
# NOTE: Create a defensive copy of the original inputs
140
- return LLMInputs(prompt_token_ids=new_token_ids,
141
- prompt=new_prompt,
142
- multi_modal_data=multi_modal_data)
140
+ return token_inputs(prompt_token_ids=new_token_ids,
141
+ prompt=new_prompt,
142
+ multi_modal_data=multi_modal_data)
143
144
145
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
vllm/model_executor/models/fuyu.py
+10
-9
@@ -27,7 +27,8 @@ from transformers import FuyuConfig, FuyuImageProcessor
27
28
from vllm.attention import AttentionMetadata
29
from vllm.config import CacheConfig, MultiModalConfig
30
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
31
+ token_inputs)
32
from vllm.model_executor.layers.linear import ColumnParallelLinear
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -149,10 +150,10 @@ def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
150
return model_image_input
151
152
152
-def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
153
- multi_modal_data = llm_inputs.get("multi_modal_data")
153
+def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
154
+ multi_modal_data = inputs.get("multi_modal_data")
155
if multi_modal_data is None or "image" not in multi_modal_data:
155
- return llm_inputs
156
+ return inputs
157
158
model_config = ctx.model_config
159
image_data = multi_modal_data["image"]
@@ -176,8 +177,8 @@ def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
177
raise TypeError(f"Invalid image type: {type(image_data)}")
178
179
# process prompts
179
- prompt = llm_inputs.get("prompt")
180
- prompt_token_ids = llm_inputs["prompt_token_ids"]
180
+ prompt = inputs.get("prompt")
181
+ prompt_token_ids = inputs["prompt_token_ids"]
182
tokenizer = cached_get_tokenizer(model_config.model)
183
# dim0 is batch_size, dim1 is subseq_size which will always be 1
184
image_input_ids: List[List[
@@ -190,9 +191,9 @@ def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
191
new_prompt_token_ids = image_input_ids + bos_token + prompt_token_ids[
192
1:] + boa_token
193
193
- return LLMInputs(prompt=new_prompt,
194
- prompt_token_ids=new_prompt_token_ids,
195
- multi_modal_data=new_multi_modal_data)
194
+ return token_inputs(prompt=new_prompt,
195
+ prompt_token_ids=new_prompt_token_ids,
196
+ multi_modal_data=new_multi_modal_data)
197
198
199
def input_mapper_for_fuyu(ctx: InputContext, data: object):
vllm/model_executor/models/internvl.py
+11
-10
@@ -17,7 +17,8 @@ from transformers import PretrainedConfig
17
18
from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, MultiModalConfig
20
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
20
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21
+ token_inputs)
22
from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24
from vllm.model_executor.models.intern_vit import InternVisionModel
@@ -276,13 +277,13 @@ class InternVLInputPipeline:
277
def input_processor(
278
self,
279
ctx: InputContext,
279
- llm_inputs: LLMInputs,
280
+ inputs: DecoderOnlyInputs,
281
*,
282
max_dynamic_patch: Optional[int] = None,
282
- ) -> LLMInputs:
283
- multi_modal_data = llm_inputs.get("multi_modal_data")
283
+ ) -> DecoderOnlyInputs:
284
+ multi_modal_data = inputs.get("multi_modal_data")
285
if multi_modal_data is None or "image" not in multi_modal_data:
285
- return llm_inputs
286
+ return inputs
287
288
model_config = ctx.model_config
289
hf_config = ctx.get_hf_config()
@@ -311,8 +312,8 @@ class InternVLInputPipeline:
312
model_config.tokenizer,
313
trust_remote_code=model_config.trust_remote_code)
314
314
- prompt = llm_inputs.get("prompt")
315
- prompt_token_ids = llm_inputs["prompt_token_ids"]
315
+ prompt = inputs.get("prompt")
316
+ prompt_token_ids = inputs["prompt_token_ids"]
317
if prompt is None:
318
prompt = tokenizer.decode(prompt_token_ids)
319
@@ -320,9 +321,9 @@ class InternVLInputPipeline:
321
num_patches)
322
new_prompt_token_ids = tokenizer.encode(new_prompt)
323
323
- return LLMInputs(prompt=prompt,
324
- prompt_token_ids=new_prompt_token_ids,
325
- multi_modal_data=multi_modal_data)
324
+ return token_inputs(prompt=prompt,
325
+ prompt_token_ids=new_prompt_token_ids,
326
+ multi_modal_data=multi_modal_data)
327
328
def input_mapper(
329
self,
vllm/model_executor/models/llava.py
+6
-6
@@ -9,7 +9,7 @@ from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
9
10
from vllm.attention import AttentionMetadata
11
from vllm.config import CacheConfig, MultiModalConfig
12
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12
+from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
13
from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -125,10 +125,10 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
125
raise NotImplementedError(msg)
126
127
128
-def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
129
- multi_modal_data = llm_inputs.get("multi_modal_data")
128
+def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
129
+ multi_modal_data = inputs.get("multi_modal_data")
130
if multi_modal_data is None or "image" not in multi_modal_data:
131
- return llm_inputs
131
+ return inputs
132
133
model_config = ctx.model_config
134
hf_config = ctx.get_hf_config(LlavaConfig)
@@ -151,7 +151,7 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
151
return input_processor_for_clip(
152
model_config,
153
vision_config,
154
- llm_inputs,
154
+ inputs,
155
image_token_id=hf_config.image_token_index,
156
image_feature_size_override=image_feature_size,
157
)
@@ -159,7 +159,7 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
159
return input_processor_for_siglip(
160
model_config,
161
vision_config,
162
- llm_inputs,
162
+ inputs,
163
image_token_id=hf_config.image_token_index,
164
image_feature_size_override=image_feature_size,
165
)
vllm/model_executor/models/llava_next.py
+7
-6
@@ -12,7 +12,7 @@ from typing_extensions import NotRequired
12
13
from vllm.attention import AttentionMetadata
14
from vllm.config import CacheConfig, MultiModalConfig
15
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15
+from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16
from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -201,10 +201,11 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
201
raise NotImplementedError(msg)
202
203
204
-def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
205
- multi_modal_data = llm_inputs.get("multi_modal_data")
204
+def input_processor_for_llava_next(ctx: InputContext,
205
+ inputs: DecoderOnlyInputs):
206
+ multi_modal_data = inputs.get("multi_modal_data")
207
if multi_modal_data is None or "image" not in multi_modal_data:
207
- return llm_inputs
208
+ return inputs
209
210
model_config = ctx.model_config
211
hf_config = ctx.get_hf_config(LlavaNextConfig)
@@ -239,7 +240,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
240
return input_processor_for_clip(
241
model_config,
242
vision_config,
242
- llm_inputs,
243
+ inputs,
244
image_token_id=hf_config.image_token_index,
245
image_feature_size_override=image_feature_size,
246
)
@@ -247,7 +248,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
248
return input_processor_for_siglip(
249
model_config,
250
vision_config,
250
- llm_inputs,
251
+ inputs,
252
image_token_id=hf_config.image_token_index,
253
image_feature_size_override=image_feature_size,
254
)
vllm/model_executor/models/llava_next_video.py
+10
-9
@@ -11,7 +11,8 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
11
12
from vllm.attention import AttentionMetadata
13
from vllm.config import CacheConfig, MultiModalConfig
14
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15
+ token_inputs)
16
from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -139,10 +140,10 @@ def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
140
141
142
def input_processor_for_llava_next_video(ctx: InputContext,
142
- llm_inputs: LLMInputs):
143
- multi_modal_data = llm_inputs.get("multi_modal_data")
143
+ inputs: DecoderOnlyInputs):
144
+ multi_modal_data = inputs.get("multi_modal_data")
145
if multi_modal_data is None or "video" not in multi_modal_data:
145
- return llm_inputs
146
+ return inputs
147
video_data = multi_modal_data["video"]
148
149
model_config = ctx.model_config
@@ -160,15 +161,15 @@ def input_processor_for_llava_next_video(ctx: InputContext,
161
162
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
163
tokenizer,
163
- llm_inputs.get("prompt"),
164
- llm_inputs["prompt_token_ids"],
164
+ inputs.get("prompt"),
165
+ inputs["prompt_token_ids"],
166
placeholder_token_id=hf_config.video_token_index,
167
repeat_count=video_feature_size,
168
)
169
169
- return LLMInputs(prompt_token_ids=new_token_ids,
170
- prompt=new_prompt,
171
- multi_modal_data=multi_modal_data)
170
+ return token_inputs(prompt_token_ids=new_token_ids,
171
+ prompt=new_prompt,
172
+ multi_modal_data=multi_modal_data)
173
174
elif is_list_of(video_data, np.ndarray):
175
raise NotImplementedError(
vllm/model_executor/models/llava_onevision.py
+20
-22
@@ -15,8 +15,8 @@ from typing_extensions import NotRequired
15
16
from vllm.attention import AttentionMetadata
17
from vllm.config import CacheConfig, MultiModalConfig
18
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
19
-from vllm.logger import init_logger
18
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
19
+ token_inputs)
20
from vllm.model_executor.layers.activation import get_act_fn
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -37,8 +37,6 @@ from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
37
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38
merge_multimodal_embeddings)
39
40
-logger = init_logger(__name__)
41
-
40
# Result in the max possible feature size (2x2 grid of 336x336px tiles)
41
MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
42
@@ -252,10 +250,10 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
250
251
252
def input_processor_when_multimodal_input_image(ctx: InputContext,
255
- llm_inputs: LLMInputs):
256
- multi_modal_data = llm_inputs.get("multi_modal_data")
253
+ inputs: DecoderOnlyInputs):
254
+ multi_modal_data = inputs.get("multi_modal_data")
255
if multi_modal_data is None or "image" not in multi_modal_data:
258
- return llm_inputs
256
+ return inputs
257
258
model_config = ctx.model_config
259
hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
@@ -290,7 +288,7 @@ def input_processor_when_multimodal_input_image(ctx: InputContext,
288
return input_processor_for_clip(
289
model_config,
290
vision_config,
293
- llm_inputs,
291
+ inputs,
292
image_token_id=hf_config.image_token_index,
293
image_feature_size_override=image_feature_size,
294
)
@@ -298,7 +296,7 @@ def input_processor_when_multimodal_input_image(ctx: InputContext,
296
return input_processor_for_siglip(
297
model_config,
298
vision_config,
301
- llm_inputs,
299
+ inputs,
300
image_token_id=hf_config.image_token_index,
301
image_feature_size_override=image_feature_size,
302
)
@@ -308,10 +306,10 @@ def input_processor_when_multimodal_input_image(ctx: InputContext,
306
307
308
def input_processor_when_multimodal_input_video(ctx: InputContext,
311
- llm_inputs: LLMInputs):
312
- multi_modal_data = llm_inputs.get("multi_modal_data")
309
+ inputs: DecoderOnlyInputs):
310
+ multi_modal_data = inputs.get("multi_modal_data")
311
if multi_modal_data is None or "video" not in multi_modal_data:
314
- return llm_inputs
312
+ return inputs
313
video_data = multi_modal_data["video"]
314
315
model_config = ctx.model_config
@@ -326,15 +324,15 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
324
325
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
326
tokenizer,
329
- llm_inputs.get("prompt"),
330
- llm_inputs["prompt_token_ids"],
327
+ inputs.get("prompt"),
328
+ inputs["prompt_token_ids"],
329
placeholder_token_id=hf_config.video_token_index,
330
repeat_count=video_feature_size,
331
)
332
335
- return LLMInputs(prompt_token_ids=new_token_ids,
336
- prompt=new_prompt,
337
- multi_modal_data=multi_modal_data)
333
+ return token_inputs(prompt_token_ids=new_token_ids,
334
+ prompt=new_prompt,
335
+ multi_modal_data=multi_modal_data)
336
337
elif is_list_of(video_data, np.ndarray):
338
raise NotImplementedError(
@@ -345,15 +343,15 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
343
344
345
def input_processor_for_llava_onevision(ctx: InputContext,
348
- llm_inputs: LLMInputs):
349
- multi_modal_data = llm_inputs.get("multi_modal_data")
346
+ inputs: DecoderOnlyInputs):
347
+ multi_modal_data = inputs.get("multi_modal_data")
348
if multi_modal_data is None or ("video" not in multi_modal_data
349
and "image" not in multi_modal_data):
352
- return llm_inputs
350
+ return inputs
351
if "image" in multi_modal_data:
354
- return input_processor_when_multimodal_input_image(ctx, llm_inputs)
352
+ return input_processor_when_multimodal_input_image(ctx, inputs)
353
if "video" in multi_modal_data:
356
- return input_processor_when_multimodal_input_video(ctx, llm_inputs)
354
+ return input_processor_when_multimodal_input_video(ctx, inputs)
355
356
msg = "Unsupported multi data type"
357
raise NotImplementedError(msg)
vllm/model_executor/models/minicpmv.py
+9
-9
@@ -36,7 +36,8 @@ from typing_extensions import NotRequired
36
37
from vllm.attention import AttentionMetadata
38
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
39
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
39
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
40
+ token_inputs)
41
from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.quantization import QuantizationConfig
43
from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
@@ -256,7 +257,7 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
257
258
259
def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
259
- return SequenceData.from_token_counts((0, seq_len))
260
+ return SequenceData.from_prompt_token_counts((0, seq_len))
261
262
263
def dummy_image_for_minicpmv(ctx: InputContext, hf_config: PretrainedConfig,
@@ -279,10 +280,10 @@ def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
280
return seq_data, mm_data
281
282
282
-def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
283
- multi_modal_data = llm_inputs.get("multi_modal_data")
283
+def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
284
+ multi_modal_data = inputs.get("multi_modal_data")
285
if multi_modal_data is None or "image" not in multi_modal_data:
285
- return llm_inputs
286
+ return inputs
287
model_config = ctx.model_config
288
version = get_version_by_config(model_config.hf_config)
289
tokenizer = cached_get_tokenizer(
@@ -297,8 +298,8 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
298
return image_processor. \
299
get_slice_image_placeholder(image_size, num_image)
300
300
- prompt = llm_inputs.get("prompt")
301
- token_ids = llm_inputs.get("prompt_token_ids")
301
+ prompt = inputs.get("prompt")
302
+ token_ids = inputs.get("prompt_token_ids")
303
if prompt is None:
304
prompt = tokenizer.decode(token_ids)
305
@@ -332,12 +333,11 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
333
_build_image_input(ctx, image) for image in images
334
]
335
335
- llm_inputs = LLMInputs(
336
+ return token_inputs(
337
prompt_token_ids=new_token_ids,
338
prompt=new_prompt,
339
multi_modal_data=multi_modal_data,
340
)
340
- return llm_inputs
341
342
343
def input_mapper_for_minicpmv(ctx: InputContext, data: object):
vllm/model_executor/models/mllama.py
+26
-26
@@ -14,7 +14,6 @@
14
# limitations under the License.
15
"""PyTorch Mllama model."""
16
import math
17
-from array import array
17
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
18
TypedDict, Union)
19
@@ -37,7 +36,8 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
36
from vllm.attention.ops.paged_attn import PagedAttention
37
from vllm.config import CacheConfig, MultiModalConfig
38
from vllm.distributed import get_tensor_model_parallel_world_size
40
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
39
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
40
+ EncoderDecoderInputs, InputContext)
41
from vllm.logger import init_logger
42
from vllm.model_executor.layers.layernorm import RMSNorm
43
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -51,7 +51,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
51
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
from vllm.multimodal import MULTIMODAL_REGISTRY
54
-from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
54
+from vllm.sequence import SequenceData
55
56
from .clip import CLIPMLP
57
from .interfaces import SupportsMultiModal
@@ -86,24 +86,24 @@ def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
86
return num_images
87
88
89
-def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
89
+def input_processor_for_mllama(ctx: InputContext,
90
+ inputs: Union[DecoderOnlyInputs,
91
+ EncoderDecoderInputs]):
92
# move encoder_prompt to prompt
91
- if llm_inputs.get("prompt") is None:
92
- llm_inputs["prompt"] = llm_inputs["encoder_prompt"]
93
- llm_inputs["prompt_token_ids"] = llm_inputs["encoder_prompt_token_ids"]
93
+ if inputs.get("prompt") is None:
94
+ inputs["prompt"] = inputs["encoder_prompt"]
95
+ inputs["prompt_token_ids"] = inputs["encoder_prompt_token_ids"]
96
97
# process multi-modal data
96
- assert "decoder_multi_modal_data" not in llm_inputs, \
97
- "multi-modal data should be put in encoder message of mllama"
98
- multi_modal_data = llm_inputs.get("encoder_multi_modal_data")
98
+ multi_modal_data = inputs.get("encoder_multi_modal_data")
99
100
if multi_modal_data is None or "image" not in multi_modal_data \
101
or multi_modal_data["image"] is None:
102
# text-only
103
- llm_inputs["encoder_prompt"] = ""
104
- llm_inputs["encoder_prompt_token_ids"] = []
105
- llm_inputs["encoder_multi_modal_data"] = {}
106
- return llm_inputs
103
+ inputs["encoder_prompt"] = ""
104
+ inputs["encoder_prompt_token_ids"] = []
105
+ inputs["encoder_multi_modal_data"] = {}
106
+ return inputs
107
108
if isinstance(multi_modal_data['image'], Image.Image):
109
multi_modal_data['image'] = [multi_modal_data['image']]
@@ -111,7 +111,7 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
111
# are attended by the decoded tokens, we only need to
112
# get the number of tiles for those images.
113
num_decode_images = _get_num_image_in_last_group(
114
- llm_inputs["prompt_token_ids"])
114
+ inputs["prompt_token_ids"])
115
hf_config = ctx.model_config.hf_config
116
num_tiles = 0
117
for image in multi_modal_data["image"][::-1]:
@@ -137,11 +137,10 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
137
"chunk size should be multiple of 14"
138
token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
139
num_tokens = num_tiles * token_per_chunk
140
- llm_inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
141
- llm_inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID
142
- ] * num_tokens
140
+ inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
141
+ inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID] * num_tokens
142
144
- return llm_inputs
143
+ return inputs
144
145
146
def get_max_mllama_image_tokens(ctx: InputContext) -> int:
@@ -154,17 +153,18 @@ def dummy_decoder_seq_data(seq_len: int, num_images: int):
153
# <|image|> * num_images + 0 * (seq_len - num_images)
154
assert seq_len >= num_images, \
155
"seq_len should be greater than or equal to num_images"
157
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
158
- [MLLAMA_IMAGE_TOKEN_ID]) * num_images
159
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * (seq_len - num_images)
160
- return SequenceData(token_ids)
156
+
157
+ return SequenceData.from_prompt_token_counts(
158
+ (MLLAMA_IMAGE_TOKEN_ID, num_images),
159
+ (0, seq_len - num_images),
160
+ )
161
162
163
def dummy_encoder_seq_data(ctx: InputContext, num_images: int):
164
num_tokens = get_max_mllama_image_tokens(ctx) * num_images
165
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
166
- [MLLAMA_IMAGE_TOKEN_ID]) * num_tokens
167
- return SequenceData(token_ids)
165
+
166
+ return SequenceData.from_prompt_token_counts(
167
+ (MLLAMA_IMAGE_TOKEN_ID, num_tokens))
168
169
170
def dummy_image(num_images: int, ):
vllm/model_executor/models/molmo.py
+8
-9
@@ -23,7 +23,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
23
get_tensor_model_parallel_world_size,
24
split_tensor_along_last_dim,
25
tensor_model_parallel_all_gather)
26
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
26
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
27
+ token_inputs)
28
from vllm.model_executor import SamplingMetadata
29
from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
30
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -945,9 +946,9 @@ def pad_images(
946
return images, image_input_idx, image_masks
947
948
948
-def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
949
- prompt = llm_inputs.get("prompt", None)
950
- multi_modal_data = llm_inputs.get("multi_modal_data", None)
949
+def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
950
+ prompt = inputs.get("prompt", None)
951
+ multi_modal_data = inputs.get("multi_modal_data", None)
952
if multi_modal_data is not None:
953
image = multi_modal_data.get("image", None)
954
else:
@@ -965,9 +966,7 @@ def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
966
elif prompt is not None:
967
out = processor.process(prompt, image)
968
else:
968
- out = processor.process(None,
969
- image,
970
- tokens=llm_inputs["prompt_token_ids"])
969
+ out = processor.process(None, image, tokens=inputs["prompt_token_ids"])
970
971
image_processor = processor.image_processor
972
max_total_crops = 1 + image_processor.max_crops
@@ -1020,9 +1019,9 @@ def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
1019
1020
multi_modal_data = dict(image=image_data)
1021
1023
- return LLMInputs(
1022
+ return token_inputs(
1023
prompt_token_ids=out["input_ids"],
1025
- prompt=llm_inputs["prompt"],
1024
+ prompt=inputs["prompt"],
1025
multi_modal_data=multi_modal_data,
1026
)
1027
vllm/model_executor/models/paligemma.py
+11
-9
@@ -7,7 +7,8 @@ from transformers import PaliGemmaConfig
7
8
from vllm.attention import AttentionMetadata
9
from vllm.config import CacheConfig, MultiModalConfig
10
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
10
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
11
+ token_inputs)
12
from vllm.logger import init_logger
13
from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -68,7 +69,8 @@ def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
69
return seq_data, mm_data
70
71
71
-def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
72
+def input_processor_for_paligemma(ctx: InputContext,
73
+ inputs: DecoderOnlyInputs):
74
75
"""
76
The correct prompt format needs to be:
@@ -77,9 +79,9 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
79
See https://github.com/huggingface/transformers/blob/25245ec26dc29bcf6102e1b4ddd0dfd02e720cf5/src/transformers/models/paligemma/processing_paligemma.py#L55
80
""" # noqa
81
80
- multi_modal_data = llm_inputs.get("multi_modal_data")
82
+ multi_modal_data = inputs.get("multi_modal_data")
83
if multi_modal_data is None or "image" not in multi_modal_data:
82
- return llm_inputs
84
+ return inputs
85
86
model_config = ctx.model_config
87
hf_config = ctx.get_hf_config(PaliGemmaConfig)
@@ -91,8 +93,8 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
93
image_token_str_pad = image_token_str * image_feature_size
94
image_token_ids_pad = [hf_config.image_token_index] * image_feature_size
95
94
- orig_prompt = llm_inputs.get("prompt")
95
- orig_prompt_ids = llm_inputs.get("prompt_token_ids")
96
+ orig_prompt = inputs.get("prompt")
97
+ orig_prompt_ids = inputs.get("prompt_token_ids")
98
99
if orig_prompt is not None and image_token_str in orig_prompt:
100
logger.warning(
@@ -106,9 +108,9 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
108
new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline
109
110
# NOTE: Create a defensive copy of the original inputs
109
- return LLMInputs(prompt_token_ids=new_token_ids,
110
- prompt=new_prompt,
111
- multi_modal_data=multi_modal_data)
111
+ return token_inputs(prompt_token_ids=new_token_ids,
112
+ prompt=new_prompt,
113
+ multi_modal_data=multi_modal_data)
114
115
116
class PaliGemmaMultiModalProjector(nn.Module):
vllm/model_executor/models/phi3v.py
+10
-10
@@ -27,7 +27,8 @@ from transformers import CLIPVisionConfig, PretrainedConfig
27
28
from vllm.attention import AttentionMetadata
29
from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
30
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
31
+ token_inputs)
32
from vllm.logger import init_logger
33
from vllm.model_executor.layers.pooler import Pooler, PoolingType
34
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -410,12 +411,12 @@ def _get_image_placeholder_token_id_candidates(
411
412
413
def input_processor_for_phi3v(ctx: InputContext,
413
- llm_inputs: LLMInputs,
414
+ inputs: DecoderOnlyInputs,
415
*,
416
num_crops: Optional[int] = None):
416
- multi_modal_data = llm_inputs.get("multi_modal_data")
417
+ multi_modal_data = inputs.get("multi_modal_data")
418
if multi_modal_data is None or "image" not in multi_modal_data:
418
- return llm_inputs
419
+ return inputs
420
421
model_config = ctx.model_config
422
hf_config = ctx.get_hf_image_processor_config()
@@ -447,7 +448,7 @@ def input_processor_for_phi3v(ctx: InputContext,
448
else:
449
raise TypeError(f"Invalid image type: {type(image_data)}")
450
450
- prompt = llm_inputs.get("prompt")
451
+ prompt = inputs.get("prompt")
452
if prompt is None:
453
# for async server request, we assume prompt and its token_ids is always
454
# in correct format. And num_image_tags == len(image_data) always True.
@@ -464,7 +465,7 @@ def input_processor_for_phi3v(ctx: InputContext,
465
image_data), "The count of image_placeholder not match image's"
466
new_prompt = prompt
467
467
- prompt_token_ids = llm_inputs["prompt_token_ids"].copy()
468
+ prompt_token_ids = inputs["prompt_token_ids"].copy()
469
470
print("prompt_token_ids (old)", prompt_token_ids)
471
@@ -506,10 +507,9 @@ def input_processor_for_phi3v(ctx: InputContext,
507
new_token_ids.append(token_id)
508
509
# NOTE: Create a defensive copy of the original inputs
509
- llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
510
- prompt=new_prompt,
511
- multi_modal_data=multi_modal_data)
512
- return llm_inputs
510
+ return token_inputs(prompt_token_ids=new_token_ids,
511
+ prompt=new_prompt,
512
+ multi_modal_data=multi_modal_data)
513
514
515
@MULTIMODAL_REGISTRY.register_image_input_mapper()
vllm/model_executor/models/pixtral.py
+7
-7
@@ -14,7 +14,7 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalMask
14
15
from vllm.attention import AttentionMetadata
16
from vllm.config import CacheConfig, MultiModalConfig
17
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
17
+from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
18
from vllm.model_executor.layers.layernorm import RMSNorm
19
from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -62,7 +62,7 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
62
image_feature_size = (size**2) // (patch_size**2)
63
64
num_image_tokens = image_feature_size * num_images
65
- seq_data = SequenceData.from_token_counts(
65
+ seq_data = SequenceData.from_prompt_token_counts(
66
(image_token_id, num_image_tokens),
67
(0, seq_len - num_image_tokens),
68
)
@@ -102,8 +102,8 @@ def input_mapper_for_pixtral(ctx: InputContext,
102
return MultiModalInputs({"images": images})
103
104
105
-def input_processor_for_pixtral(ctx: InputContext, llm_inputs: LLMInputs):
106
- multi_modal_data = llm_inputs.get("multi_modal_data")
105
+def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
106
+ multi_modal_data = inputs.get("multi_modal_data")
107
if multi_modal_data is not None and "image" in multi_modal_data:
108
tokenizer = cached_get_tokenizer(
109
ctx.model_config.tokenizer,
@@ -112,15 +112,15 @@ def input_processor_for_pixtral(ctx: InputContext, llm_inputs: LLMInputs):
112
mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
113
image_token_id = mm_encoder.special_ids.img
114
115
- if image_token_id not in llm_inputs['prompt_token_ids']:
115
+ if image_token_id not in inputs['prompt_token_ids']:
116
raise ValueError(
117
- (f"You've passed {llm_inputs=} without {image_token_id=}"
117
+ (f"You've passed {inputs=} without {image_token_id=}"
118
" Make sure to process your input via mistral_common's"
119
" tokenizer or pass a chat completion request. For more"
120
" For more info, see: "
121
"https://github.com/vllm-project/vllm/issues/8411."))
122
123
- return llm_inputs
123
+ return inputs
124
125
126
@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral)
vllm/model_executor/models/qwen.py
+13
-12
@@ -22,7 +22,8 @@ from transformers import PretrainedConfig
22
from vllm.attention import Attention, AttentionMetadata
23
from vllm.config import CacheConfig, MultiModalConfig
24
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
25
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
25
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
26
+ token_inputs)
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
29
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -652,30 +653,30 @@ def get_image_text(image_num: int, padding: bool) -> str:
653
654
655
def input_processor_for_qwen(ctx: InputContext,
655
- llm_inputs: LLMInputs) -> LLMInputs:
656
+ inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
657
"""Processes the inputs, which may or may not be multimodal.
658
Multimodal inputs will only be processed if the model has a "visual"
659
component in its model config, otherwise they'll be ignored.
660
661
Args:
662
ctx: Context of the loaded model.
662
- llm_inputs: LLM inputs which may have a multi_modal_data attribute.
663
+ inputs: LLM inputs which may have a multi_modal_data attribute.
664
665
Returns:
666
If the model is language only or not multimodal inputs were provided,
666
- returns llm_inputs unmodified. Otherwise, processes the multimodal
667
+ returns inputs unmodified. Otherwise, processes the multimodal
668
images / image embeddings and adds the fixed-length image placeholders.
669
"""
669
- multi_modal_data = llm_inputs.get("multi_modal_data")
670
+ multi_modal_data = inputs.get("multi_modal_data")
671
672
# Only process images if we have multimodal data and a visual config
673
hf_config = ctx.get_hf_config()
674
if (multi_modal_data is None or "image" not in multi_modal_data
675
or not hasattr(hf_config, "visual")):
675
- return llm_inputs
676
+ return inputs
677
677
- prompt = llm_inputs.get("prompt")
678
- prompt_token_ids = llm_inputs["prompt_token_ids"]
678
+ prompt = inputs.get("prompt")
679
+ prompt_token_ids = inputs["prompt_token_ids"]
680
model_config = ctx.model_config
681
tokenizer = cached_get_tokenizer(
682
model_config.tokenizer,
@@ -713,9 +714,9 @@ def input_processor_for_qwen(ctx: InputContext,
714
715
new_prompt_token_ids = tokenizer.encode(new_prompt)
716
716
- return LLMInputs(prompt=new_prompt,
717
- prompt_token_ids=new_prompt_token_ids,
718
- multi_modal_data=multi_modal_data)
717
+ return token_inputs(prompt=new_prompt,
718
+ prompt_token_ids=new_prompt_token_ids,
719
+ multi_modal_data=multi_modal_data)
720
721
722
def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
@@ -822,7 +823,7 @@ def dummy_data_for_qwen(
823
# The presence of a visual config indicates this is a multimodal model.
824
# If we don't have it, the model is considered an LLM for warmup purposes.
825
if not hasattr(hf_config, "visual"):
825
- seq_data = SequenceData.from_token_counts((0, seq_len))
826
+ seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
827
mm_data = None
828
return seq_data, mm_data
829
vllm/model_executor/models/qwen2_vl.py
+14
-11
@@ -46,7 +46,8 @@ from vllm.attention.selector import (_Backend, backend_name_to_enum,
46
from vllm.config import CacheConfig, MultiModalConfig
47
from vllm.distributed import get_pp_group, parallel_state
48
from vllm.distributed import utils as dist_utils
49
-from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
49
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
50
+ token_inputs)
51
from vllm.logger import init_logger
52
from vllm.model_executor import SamplingMetadata
53
from vllm.model_executor.layers.activation import QuickGELU
@@ -716,7 +717,7 @@ def dummy_data_for_qwen2_vl(
717
718
hf_config = ctx.get_hf_config(Qwen2VLConfig)
719
719
- dummy_seqdata = SequenceData.from_token_counts(
720
+ dummy_seqdata = SequenceData.from_prompt_token_counts(
721
(hf_config.vision_start_token_id, 1),
722
(hf_config.image_token_id, max_llm_image_tokens),
723
(hf_config.vision_end_token_id, 1),
@@ -799,11 +800,13 @@ def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
800
return prompt_token_ids_with_data
801
802
802
-def input_processor_for_qwen2_vl(ctx: InputContext,
803
- llm_inputs: LLMInputs) -> LLMInputs:
804
- multi_modal_data = llm_inputs.get("multi_modal_data", None)
803
+def input_processor_for_qwen2_vl(
804
+ ctx: InputContext,
805
+ inputs: DecoderOnlyInputs,
806
+) -> DecoderOnlyInputs:
807
+ multi_modal_data = inputs.get("multi_modal_data", None)
808
if multi_modal_data is None:
806
- return llm_inputs
809
+ return inputs
810
811
image_inputs = multi_modal_data.get("image", None)
812
video_inputs = multi_modal_data.get("video", None)
@@ -817,7 +820,7 @@ def input_processor_for_qwen2_vl(ctx: InputContext,
820
# `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2VLProcessor`.
821
#
822
# The following code is equivalent to:
820
- # prompt = llm_inputs["prompt"]
823
+ # prompt = inputs["prompt"]
824
# inputs = processor(text=[prompt],
825
# images=image_inputs,
826
# videos=video_inputs,
@@ -825,9 +828,9 @@ def input_processor_for_qwen2_vl(ctx: InputContext,
828
# return_tensors="pt")
829
# prompt_token_ids = inputs["input_ids"][0].tolist()
830
828
- prompt_token_ids = llm_inputs.get("prompt_token_ids", None)
831
+ prompt_token_ids = inputs.get("prompt_token_ids", None)
832
if prompt_token_ids is None:
830
- prompt = llm_inputs["prompt"]
833
+ prompt = inputs["prompt"]
834
prompt_token_ids = processor.tokenizer(
835
prompt,
836
padding=True,
@@ -868,9 +871,9 @@ def input_processor_for_qwen2_vl(ctx: InputContext,
871
image_processor,
872
prompt_token_ids)
873
871
- return LLMInputs(
874
+ return token_inputs(
875
prompt_token_ids=prompt_token_ids,
873
- prompt=llm_inputs["prompt"],
876
+ prompt=inputs["prompt"],
877
multi_modal_data=multi_modal_data,
878
)
879
vllm/model_executor/models/siglip.py
+8
-8
@@ -13,7 +13,7 @@ from transformers.models.siglip.modeling_siglip import SiglipSdpaAttention
13
14
from vllm.config import ModelConfig
15
from vllm.distributed import divide, get_tensor_model_parallel_world_size
16
-from vllm.inputs import LLMInputs
16
+from vllm.inputs import DecoderOnlyInputs, token_inputs
17
from vllm.model_executor.layers.activation import get_act_fn
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
QKVParallelLinear,
@@ -67,7 +67,7 @@ def dummy_seq_data_for_siglip(
67
else:
68
image_feature_size = image_feature_size_override
69
70
- return SequenceData.from_token_counts(
70
+ return SequenceData.from_prompt_token_counts(
71
(image_token_id, image_feature_size * num_images),
72
(0, seq_len - image_feature_size * num_images),
73
)
@@ -111,14 +111,14 @@ def dummy_video_for_siglip(
111
def input_processor_for_siglip(
112
model_config: ModelConfig,
113
hf_config: SiglipVisionConfig,
114
- llm_inputs: LLMInputs,
114
+ inputs: DecoderOnlyInputs,
115
*,
116
image_token_id: int,
117
image_feature_size_override: Optional[Union[int, List[int]]] = None,
118
):
119
- multi_modal_data = llm_inputs.get("multi_modal_data")
119
+ multi_modal_data = inputs.get("multi_modal_data")
120
if multi_modal_data is None or "image" not in multi_modal_data:
121
- return llm_inputs
121
+ return inputs
122
123
tokenizer = cached_get_tokenizer(model_config.tokenizer)
124
@@ -135,14 +135,14 @@ def input_processor_for_siglip(
135
136
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
137
tokenizer,
138
- llm_inputs.get("prompt"),
139
- llm_inputs["prompt_token_ids"],
138
+ inputs.get("prompt"),
139
+ inputs["prompt_token_ids"],
140
placeholder_token_id=image_token_id,
141
repeat_count=image_feature_size,
142
)
143
144
# NOTE: Create a defensive copy of the original inputs
145
- return LLMInputs(
145
+ return token_inputs(
146
prompt_token_ids=new_token_ids,
147
prompt=new_prompt,
148
multi_modal_data=multi_modal_data,
vllm/model_executor/models/ultravox.py
+9
-9
@@ -18,7 +18,7 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
18
from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, MultiModalConfig
20
from vllm.inputs import INPUT_REGISTRY
21
-from vllm.inputs.data import LLMInputs
21
+from vllm.inputs.data import DecoderOnlyInputs, token_inputs
22
from vllm.inputs.registry import InputContext
23
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
24
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -156,10 +156,10 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
156
return MultiModalInputs({"audio_features": audio_features})
157
158
159
-def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
160
- multi_modal_data = llm_inputs.get("multi_modal_data")
159
+def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
160
+ multi_modal_data = inputs.get("multi_modal_data")
161
if multi_modal_data is None or "audio" not in multi_modal_data:
162
- return llm_inputs
162
+ return inputs
163
164
feature_extractor = whisper_feature_extractor(ctx)
165
audios = multi_modal_data["audio"]
@@ -196,16 +196,16 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
196
197
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
198
tokenizer,
199
- llm_inputs.get("prompt"),
200
- llm_inputs["prompt_token_ids"],
199
+ inputs.get("prompt"),
200
+ inputs["prompt_token_ids"],
201
placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN,
202
repeat_count=audio_token_counts,
203
)
204
205
# NOTE: Create a defensive copy of the original inputs
206
- return LLMInputs(prompt_token_ids=new_token_ids,
207
- prompt=new_prompt,
208
- multi_modal_data=multi_modal_data)
206
+ return token_inputs(prompt_token_ids=new_token_ids,
207
+ prompt=new_prompt,
208
+ multi_modal_data=multi_modal_data)
209
210
211
class StackAudioFrames(nn.Module):
vllm/sequence.py
+47
-27
@@ -13,8 +13,7 @@ from typing import Set, Tuple, Union, cast
13
import msgspec
14
import torch
15
16
-from vllm.inputs import EncoderDecoderLLMInputs, LLMInputs
17
-from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
16
+from vllm.inputs.parse import is_encoder_decoder_inputs
17
from vllm.lora.request import LoRARequest
18
from vllm.pooling_params import PoolingParams
19
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -22,6 +21,7 @@ from vllm.sampling_params import SamplingParams
21
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
22
23
if TYPE_CHECKING:
24
+ from vllm.inputs import SingletonInputs
25
from vllm.multimodal.base import MultiModalDataDict
26
27
VLLM_TOKEN_ID_ARRAY_TYPE = "l"
@@ -29,6 +29,11 @@ VLLM_TOKEN_ID_ARRAY_TYPE = "l"
29
VLLM_INVALID_TOKEN_ID = -1
30
31
32
+def array_full(token_id: int, count: int):
33
+ """:class:`array` equivalent of :func:`numpy.full`."""
34
+ return array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
35
+
36
+
37
# We use dataclass for now because it is used for
38
# openai server output, and msgspec is not serializable.
39
# TODO(sang): Fix it.
@@ -173,22 +178,34 @@ class SequenceData(msgspec.Struct,
178
_mrope_position_delta: Optional[int] = None
179
180
@staticmethod
176
- def from_token_counts(*token_counts: Tuple[int, int]) -> "SequenceData":
181
+ def from_prompt_token_counts(
182
+ *token_counts: Tuple[int, int]) -> "SequenceData":
183
+ """
184
+ Construct a :class:`SequenceData` instance by concatenating
185
+ prompt token sequences.
186
+
187
+ Each tuple represents one token sequence, expressed in the form
188
+ :code:`(token_id, count)`.
189
+ """
190
if len(token_counts) == 0:
191
return SequenceData.from_seqs([])
192
180
- arrs = [
181
- array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
182
- for token_id, count in token_counts
183
- ]
193
+ prompt_token_ids_arr = reduce(
194
+ array.__iadd__,
195
+ (array_full(token_id, count) for token_id, count in token_counts),
196
+ )
197
185
- return SequenceData(reduce(array.__add__, arrs))
198
+ return SequenceData(prompt_token_ids_arr)
199
200
@staticmethod
201
def from_seqs(
202
prompt_token_ids: GenericSequence[int],
203
output_token_ids: Optional[GenericSequence[int]] = None,
204
) -> "SequenceData":
205
+ """
206
+ Construct a :class:`SequenceData` instance from prompt and output
207
+ token sequences.
208
+ """
209
prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
210
prompt_token_ids)
211
@@ -362,14 +379,14 @@ class SequenceData(msgspec.Struct,
379
class Sequence:
380
"""Stores the data, status, and block information of a sequence.
381
365
- The sequence is constructed from the LLMInputs instance passed
366
- in through the `inputs` constructor argument.
382
+ The sequence is constructed from the :code:`SingletonInputs` instance
383
+ passed in through the :code:`inputs` constructor argument.
384
368
- For encoder/decoder models, LLMInputs encapsulates both a
385
+ For encoder/decoder models, SingletonInputs encapsulates both a
386
decoder and encoder prompt, creating an ambiguity about which
387
prompt to construct the sequence from. The `from_decoder_prompt`
388
constructor argument signals whether to construct the Sequence
372
- from the LLMInputs decoder prompt, or encoder prompt.
389
+ from the SingletonInputs decoder prompt, or encoder prompt.
390
391
Args:
392
seq_id: The ID of the sequence.
@@ -379,16 +396,16 @@ class Sequence:
396
eos_token_id: The end-of-sequence (EOS) token id recognized by this LLM.
397
lora_request: LoRA request.
398
prompt_adapter_request: Prompt Adapter request.
382
- from_decoder_prompt: Construct Sequence from LLMInputs decoder prompt
383
- (True) or encoder prompt (False.) Must be True
384
- for decoder-only model.
399
+ from_decoder_prompt: Construct Sequence from SingletonInputs decoder
400
+ prompt (True) or encoder prompt (False.) Must be
401
+ True for decoder-only model.
402
403
"""
404
405
def __init__(
406
self,
407
seq_id: int,
391
- inputs: "LLMInputs",
408
+ inputs: "SingletonInputs",
409
block_size: int,
410
eos_token_id: Optional[int] = None,
411
lora_request: Optional[LoRARequest] = None,
@@ -404,19 +421,19 @@ class Sequence:
421
self.from_decoder_prompt = from_decoder_prompt
422
423
# For decoder-only models, a Sequence is constructed
407
- # from an LLMInputs instance (the `inputs` arg.)
424
+ # from an DecoderOnlyInputs instance (the `inputs` arg.)
425
#
426
# For encoder/decoder models the same `inputs`
427
# instance could be utilized to construct either an
428
# encoder sequence or a decoder sequence, because
412
- # `LLMInputs` has both decoder- and encoder-oriented
429
+ # `DecoderOnlyInputs` has both decoder- and encoder-oriented
430
# member variables (i.e. it encapsulates both an encoder
431
# and a decoder prompt.) The decision of which type of sequence
432
# to generate is determined by the `from_decoder_prompt` argument.
433
#
434
# When constructing a encoder sequence
435
# (`from_decoder_prompt` False) it matters that
419
- # the `LLMInputs` instance stored in `inputs` is valid
436
+ # the `DecoderOnlyInputs` instance stored in `inputs` is valid
437
# in the sense that its encoder-related member variables are
438
# populated; below, an exception is raised if this is
439
# not the case.
@@ -424,8 +441,7 @@ class Sequence:
441
# When constructing a decoder sequence (`from_decoder_prompt` True)
442
# it does not matter whether `inputs` has its encoder-related
443
# member variables populated.
427
- if not (from_decoder_prompt
428
- or is_valid_encoder_decoder_llm_inputs(inputs)):
444
+ if not (from_decoder_prompt or is_encoder_decoder_inputs(inputs)):
445
raise ValueError("Cannot extract encoder input prompt from "
446
f"invalid input {inputs}; did you forget the "
447
"encoder input prompt fields?")
@@ -471,15 +487,19 @@ class Sequence:
487
488
@property
489
def multi_modal_data(self) -> "MultiModalDataDict":
474
- if self.inputs.get("multi_modal_data") and self.inputs.get(
475
- "encoder_multi_modal_data"):
490
+ inputs = self.inputs
491
+
492
+ if (inputs.get("multi_modal_data")
493
+ and inputs.get("encoder_multi_modal_data")):
494
raise ValueError(
495
"Multi-modal data in both encoder and decoder is not supported."
496
)
479
- inputs = self.inputs
480
- return self.inputs.get("multi_modal_data") or (cast(
481
- EncoderDecoderLLMInputs,
482
- inputs).get("encoder_multi_modal_data")) or {}
497
+
498
+ return cast(
499
+ "MultiModalDataDict",
500
+ (inputs.get("multi_modal_data")
501
+ or inputs.get("encoder_multi_modal_data") or {}),
502
+ )
503
504
@property
505
def mm_processor_kwargs(self) -> Dict[str, Any]: