A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Core] Make encoder-decoder inputs a nested structure to be more composable (#9604)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 5, 2024 at 10:07 UTC
bbc3619dc806b25fd5e14eef90819052ab76e1c6
14 files changed
+372
-349
tests/core/utils.py
+26
-31
@@ -4,6 +4,7 @@ from typing import Sequence as GenericSequence
4
from typing import Tuple
5
6
from vllm import SamplingParams
7
+from vllm.inputs import EncoderDecoderInputs, token_inputs
8
from vllm.lora.request import LoRARequest
9
from vllm.sequence import Logprob, Sequence, SequenceGroup
10
@@ -27,10 +28,7 @@ def create_dummy_prompt(
28
prompt_tokens = list(range(prompt_length))
29
prompt_str = " ".join([str(t) for t in prompt_tokens])
30
prompt = Sequence(int(request_id),
30
- inputs={
31
- "prompt": prompt_str,
32
- "prompt_token_ids": prompt_tokens,
33
- },
31
+ inputs=token_inputs(prompt_tokens, prompt=prompt_str),
32
block_size=block_size)
33
seq_group = SequenceGroup(request_id=request_id,
34
seqs=[prompt],
@@ -63,23 +61,21 @@ def create_dummy_prompt_encoder_decoder(
61
encoder_prompt_tokens = list(reversed(list(range(encoder_prompt_length))))
62
encoder_prompt_str = " ".join([str(t) for t in encoder_prompt_tokens])
63
66
- inputs = {
67
- "prompt": decoder_prompt_str,
68
- "prompt_token_ids": decoder_prompt_tokens,
69
- "encoder_prompt": encoder_prompt_str,
70
- "encoder_prompt_token_ids": encoder_prompt_tokens,
71
- "multi_modal_data": None,
64
+ inputs: EncoderDecoderInputs = {
65
+ "decoder": token_inputs(decoder_prompt_tokens,
66
+ prompt=decoder_prompt_str),
67
+ "encoder": token_inputs(encoder_prompt_tokens,
68
+ prompt=encoder_prompt_str),
69
}
70
71
decoder_prompt = Sequence(int(request_id),
75
- inputs=inputs,
76
- block_size=block_size,
77
- from_decoder_prompt=True)
72
+ inputs=inputs["decoder"],
73
+ block_size=block_size)
74
75
encoder_prompt = Sequence(int(request_id),
80
- inputs=inputs,
81
- block_size=block_size,
82
- from_decoder_prompt=False)
76
+ inputs=inputs["encoder"],
77
+ block_size=block_size)
78
+
79
seq_group = SequenceGroup(request_id=request_id,
80
seqs=[decoder_prompt],
81
sampling_params=SamplingParams(best_of=best_of),
@@ -108,7 +104,7 @@ def create_seq_group(
104
for seq_id_offset, output_len in enumerate(seq_output_lens):
105
seq = Sequence(
106
seq_id=seq_id_start + seq_id_offset,
111
- inputs={"prompt_token_ids": prompt_token_ids},
107
+ inputs=token_inputs(prompt_token_ids),
108
block_size=16,
109
)
110
@@ -143,21 +139,19 @@ def create_seq_group_encoder_decoder(
139
140
prompt_token_ids = [0] * seq_prompt_len
141
146
- inputs = {
147
- "prompt": "",
148
- "prompt_token_ids": prompt_token_ids,
149
- "encoder_prompt": "",
150
- "encoder_prompt_token_ids": prompt_token_ids,
151
- "multi_modal_data": None,
142
+ inputs: EncoderDecoderInputs = {
143
+ "decoder": token_inputs(prompt_token_ids),
144
+ "encoder": token_inputs(prompt_token_ids),
145
}
146
147
seqs = []
148
for seq_id_offset, output_len in enumerate(seq_output_lens):
149
# Construct decoder input sequences
157
- seq = Sequence(seq_id=seq_id_start + seq_id_offset,
158
- inputs=inputs,
159
- block_size=16,
160
- from_decoder_prompt=True)
150
+ seq = Sequence(
151
+ seq_id=seq_id_start + seq_id_offset,
152
+ inputs=inputs["decoder"],
153
+ block_size=16,
154
+ )
155
156
for i in range(output_len):
157
seq.append_token_id(
@@ -167,10 +161,11 @@ def create_seq_group_encoder_decoder(
161
seqs.append(seq)
162
163
# Encoder input sequence
170
- encoder_seq = Sequence(seq_id=seq_id_start + len(seq_output_lens),
171
- inputs=inputs,
172
- block_size=16,
173
- from_decoder_prompt=False)
164
+ encoder_seq = Sequence(
165
+ seq_id=seq_id_start + len(seq_output_lens),
166
+ inputs=inputs["encoder"],
167
+ block_size=16,
168
+ )
169
170
return SequenceGroup(request_id=request_id,
171
seqs=seqs,
tests/engine/output_processor/test_stop_checker.py
+2
-1
@@ -4,6 +4,7 @@ import pytest
4
from transformers import PreTrainedTokenizer
5
6
from vllm.engine.output_processor.stop_checker import StopChecker
7
+from vllm.inputs import token_inputs
8
from vllm.sampling_params import SamplingParams
9
from vllm.sequence import Logprob, Sequence, SequenceStatus
10
@@ -15,7 +16,7 @@ def sequence_with_eos(text: str, eos_token: str,
16
"""
17
seq = Sequence(
18
seq_id=0,
18
- inputs={"prompt_token_ids": []},
19
+ inputs=token_inputs([]),
20
block_size=16,
21
eos_token_id=eos_token_id,
22
)
tests/test_cache_block_hashing.py
+3
-4
@@ -6,6 +6,7 @@ from typing import List, Optional
6
7
import pytest
8
9
+from vllm.inputs import token_inputs
10
from vllm.lora.request import LoRARequest
11
from vllm.sequence import Sequence
12
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -70,10 +71,8 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
71
hashes[-1].append([])
72
prompt_token_ids = tokenizer.encode(prompt)
73
seq = Sequence(seq_id,
73
- inputs={
74
- "prompt": prompt,
75
- "prompt_token_ids": prompt_token_ids,
76
- },
74
+ inputs=token_inputs(prompt_token_ids,
75
+ prompt=prompt),
76
block_size=block_size,
77
eos_token_id=tokenizer.tokenizer.eos_token_id,
78
lora_request=lora_request)
tests/tokenization/test_detokenize.py
+2
-4
@@ -3,6 +3,7 @@ from typing import Any, Dict, Generator, List, Optional
3
import pytest
4
from transformers import AutoTokenizer
5
6
+from vllm.inputs import token_inputs
7
from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
8
from vllm.transformers_utils.detokenizer import (Detokenizer,
9
detokenize_incrementally)
@@ -169,10 +170,7 @@ def create_sequence(prompt_token_ids=None):
170
prompt_token_ids = prompt_token_ids or [1]
171
return Sequence(
172
seq_id=0,
172
- inputs={
173
- "prompt": "<s>",
174
- "prompt_token_ids": prompt_token_ids,
175
- },
173
+ inputs=token_inputs(prompt_token_ids, prompt="<s>"),
174
block_size=16,
175
)
176
vllm/engine/llm_engine.py
+24
-27
@@ -10,7 +10,7 @@ from typing import Sequence as GenericSequence
10
from typing import Set, Type, Union, cast, overload
11
12
import torch
13
-from typing_extensions import TypeIs, TypeVar
13
+from typing_extensions import TypeVar
14
15
import vllm.envs as envs
16
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
@@ -29,9 +29,9 @@ from vllm.entrypoints.openai.logits_processors import (
29
from vllm.executor.executor_base import ExecutorBase
30
from vllm.executor.gpu_executor import GPUExecutor
31
from vllm.executor.ray_utils import initialize_ray_cluster
32
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
33
- EncoderDecoderInputs, InputRegistry, PromptType,
34
- TokensPrompt)
32
+from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
33
+ PromptType)
34
+from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt
35
from vllm.inputs.preprocess import InputPreprocessor
36
from vllm.logger import init_logger
37
from vllm.logits_process import get_bad_words_logits_processors
@@ -638,7 +638,7 @@ class LLMEngine:
638
def _add_processed_request(
639
self,
640
request_id: str,
641
- processed_inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
641
+ processed_inputs: ProcessorInputs,
642
params: Union[SamplingParams, PoolingParams],
643
arrival_time: float,
644
lora_request: Optional[LoRARequest],
@@ -669,18 +669,19 @@ class LLMEngine:
669
seq_id = next(self.seq_counter)
670
eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
671
672
- seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
672
+ if is_encoder_decoder_inputs(processed_inputs):
673
+ decoder_inputs = processed_inputs["decoder"]
674
+ encoder_inputs = processed_inputs["encoder"]
675
+ else:
676
+ decoder_inputs = processed_inputs
677
+ encoder_inputs = None
678
+
679
+ seq = Sequence(seq_id, decoder_inputs, block_size, eos_token_id,
680
lora_request, prompt_adapter_request)
681
675
- encoder_seq = None
676
- if 'encoder_prompt_token_ids' in processed_inputs:
677
- encoder_seq = Sequence(seq_id,
678
- processed_inputs,
679
- block_size,
680
- eos_token_id,
681
- lora_request,
682
- prompt_adapter_request,
683
- from_decoder_prompt=False)
682
+ encoder_seq = (None if encoder_inputs is None else Sequence(
683
+ seq_id, encoder_inputs, block_size, eos_token_id, lora_request,
684
+ prompt_adapter_request))
685
686
# Create a SequenceGroup based on SamplingParams or PoolingParams
687
if isinstance(params, SamplingParams):
@@ -874,7 +875,7 @@ class LLMEngine:
875
# This needs to happen before multimodal input pre-processing, which
876
# may add dummy <image> tokens that aren't part of the tokenizer's
877
# vocabulary.
877
- if self._is_token_prompt(prompt):
878
+ if is_token_prompt(prompt):
879
prompt_ids = prompt["prompt_token_ids"]
880
if len(prompt_ids) == 0:
881
# Empty prompt check is handled later
@@ -884,10 +885,6 @@ class LLMEngine:
885
raise ValueError(
886
"Token id {} is out of vocabulary".format(max_input_id))
887
887
- @staticmethod
888
- def _is_token_prompt(prompt: PromptType) -> TypeIs[TokensPrompt]:
889
- return isinstance(prompt, dict) and "prompt_token_ids" in prompt
890
-
888
def _create_sequence_group_with_sampling(
889
self,
890
request_id: str,
@@ -1978,17 +1975,17 @@ class LLMEngine:
1975
def is_encoder_decoder_model(self):
1976
return self.input_preprocessor.is_encoder_decoder_model()
1977
1981
- def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1982
- EncoderDecoderInputs],
1978
+ def _validate_model_inputs(self, inputs: ProcessorInputs,
1979
lora_request: Optional[LoRARequest]):
1984
- if self.model_config.is_multimodal_model:
1980
+ if is_encoder_decoder_inputs(inputs):
1981
# For encoder-decoder multimodal models, the max_prompt_len
1982
# restricts the decoder prompt length
1987
- prompt_ids = inputs.get("prompt_token_ids")
1988
- elif self.is_encoder_decoder_model():
1989
- prompt_ids = inputs.get("encoder_prompt_token_ids")
1983
+ prompt_inputs = inputs["decoder" if self.model_config.
1984
+ is_multimodal_model else "encoder"]
1985
else:
1991
- prompt_ids = inputs.get("prompt_token_ids")
1986
+ prompt_inputs = inputs
1987
+
1988
+ prompt_ids = prompt_inputs.get("prompt_token_ids")
1989
1990
if prompt_ids is None or len(prompt_ids) == 0:
1991
raise ValueError("Prompt cannot be empty")
vllm/engine/protocol.py
+16
-7
@@ -1,11 +1,12 @@
1
import asyncio
2
from abc import ABC, abstractmethod
3
-from typing import AsyncGenerator, List, Mapping, Optional, Union
3
+from typing import AsyncGenerator, List, Mapping, Optional
4
5
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
6
from vllm.config import DecodingConfig, ModelConfig
7
from vllm.core.scheduler import SchedulerOutputs
8
from vllm.inputs.data import PromptType, TokensPrompt
9
+from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
10
from vllm.inputs.preprocess import InputPreprocessor
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
@@ -60,7 +61,7 @@ class EngineClient(ABC):
61
62
async def beam_search(
63
self,
63
- prompt: Union[PromptType, List[int]],
64
+ prompt: PromptType,
65
model_config: ModelConfig,
66
request_id: str,
67
params: BeamSearchParams,
@@ -76,11 +77,19 @@ class EngineClient(ABC):
77
tokenizer = await self.get_tokenizer()
78
input_preprocessor = InputPreprocessor(model_config, tokenizer)
79
79
- (prompt_text, prompt_token_ids, multi_modal_data,
80
- mm_processor_kwargs) = input_preprocessor._extract_prompt_components(
81
- prompt,
82
- request_id=request_id,
83
- )
80
+ if is_explicit_encoder_decoder_prompt(prompt):
81
+ raise NotImplementedError
82
+ else:
83
+ processed_inputs = input_preprocessor._prompt_to_llm_inputs(
84
+ prompt,
85
+ request_id=request_id,
86
+ )
87
+
88
+ prompt_token_ids = processed_inputs["prompt_token_ids"]
89
+ prompt_text = processed_inputs.get("prompt")
90
+ multi_modal_data = processed_inputs.get("multi_modal_data")
91
+ mm_processor_kwargs = processed_inputs.get("mm_processor_kwargs")
92
+
93
tokenized_length = len(prompt_token_ids)
94
95
sort_beams_key = create_sort_beams_key_function(
vllm/inputs/__init__.py
+6
-5
@@ -1,8 +1,8 @@
1
from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
2
- ExplicitEncoderDecoderPrompt, PromptType, SingletonInputs,
3
- SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
4
- build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
5
- token_inputs, zip_enc_dec_prompts)
2
+ ExplicitEncoderDecoderPrompt, ProcessorInputs, PromptType,
3
+ SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
4
+ TokensPrompt, build_explicit_enc_dec_prompt,
5
+ to_enc_dec_tuple_list, token_inputs, zip_enc_dec_prompts)
6
from .registry import DummyData, InputContext, InputRegistry
7
8
INPUT_REGISTRY = InputRegistry()
@@ -22,9 +22,10 @@ __all__ = [
22
"ExplicitEncoderDecoderPrompt",
23
"TokenInputs",
24
"token_inputs",
25
- "SingletonInputs",
25
"DecoderOnlyInputs",
26
"EncoderDecoderInputs",
27
+ "ProcessorInputs",
28
+ "SingletonInputs",
29
"build_explicit_enc_dec_prompt",
30
"to_enc_dec_tuple_list",
31
"zip_enc_dec_prompts",
vllm/inputs/data.py
+25
-26
@@ -1,4 +1,4 @@
1
-from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
1
+from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List, Literal,
2
Optional, Tuple, Union, cast)
3
4
from typing_extensions import NotRequired, TypedDict, TypeVar
@@ -122,27 +122,30 @@ both decoder-only and encoder/decoder input types:
122
123
class TokenInputs(TypedDict):
124
"""Represents token-based inputs."""
125
+
126
+ type: Literal["token"]
127
+ """The type of inputs."""
128
+
129
prompt_token_ids: List[int]
130
"""The token IDs of the prompt."""
131
128
- prompt: NotRequired[Optional[str]]
132
+ prompt: NotRequired[str]
133
"""
134
The original prompt text corresponding to the token IDs, if available.
135
"""
136
133
- multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
137
+ multi_modal_data: NotRequired["MultiModalDataDict"]
138
"""
139
Optional multi-modal data to pass to the model,
140
if the model supports it.
141
"""
142
139
- multi_modal_placeholders: NotRequired[
140
- Optional["MultiModalPlaceholderDict"]]
143
+ multi_modal_placeholders: NotRequired["MultiModalPlaceholderDict"]
144
"""
145
Placeholder ranges for the multi-modal data.
146
"""
147
145
- mm_processor_kwargs: NotRequired[Optional[Dict[str, Any]]]
148
+ mm_processor_kwargs: NotRequired[Dict[str, Any]]
149
"""
150
Optional multi-modal processor kwargs to be forwarded to the
151
multimodal input mapper & processor. Note that if multiple modalities
@@ -159,7 +162,7 @@ def token_inputs(
162
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
163
) -> TokenInputs:
164
"""Construct :class:`TokenInputs` from optional values."""
162
- inputs = TokenInputs(prompt_token_ids=prompt_token_ids)
165
+ inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
166
167
if prompt is not None:
168
inputs["prompt"] = prompt
@@ -173,12 +176,6 @@ def token_inputs(
176
return inputs
177
178
176
-SingletonInputs = TokenInputs
177
-"""
178
-A processed :class:`SingletonPrompt` which can be passed to
179
-:class:`vllm.sequence.Sequence`.
180
-"""
181
-
179
DecoderOnlyInputs = TokenInputs
180
"""
181
The inputs in :class:`~vllm.LLMEngine` before they are
@@ -187,28 +184,30 @@ This specifies the data required for decoder-only models.
184
"""
185
186
190
-class EncoderDecoderInputs(TokenInputs):
187
+class EncoderDecoderInputs(TypedDict):
188
"""
189
The inputs in :class:`~vllm.LLMEngine` before they are
190
passed to the model executor.
191
192
This specifies the required data for encoder-decoder models.
193
"""
197
- encoder_prompt_token_ids: List[int]
198
- """The token IDs of the encoder prompt."""
194
+ encoder: TokenInputs
195
+ """The inputs for the encoder portion."""
196
200
- encoder_prompt: NotRequired[Optional[str]]
201
- """
202
- The original encoder prompt text corresponding to the token IDs, if
203
- available.
204
- """
197
+ decoder: TokenInputs
198
+ """The inputs for the decoder portion."""
199
206
- encoder_multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
207
- """
208
- Optional multi-modal data to pass to the encoder model,
209
- if the model supports it.
210
- """
200
201
+SingletonInputs = TokenInputs
202
+"""
203
+A processed :class:`SingletonPrompt` which can be passed to
204
+:class:`vllm.sequence.Sequence`.
205
+"""
206
+
207
+ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
208
+"""
209
+The inputs to :data:`vllm.inputs.InputProcessor`.
210
+"""
211
212
_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
213
_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
vllm/inputs/parse.py
+9
-6
@@ -4,9 +4,9 @@ from typing_extensions import TypeIs
4
5
from vllm.utils import is_list_of
6
7
-from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
8
- ExplicitEncoderDecoderPrompt, PromptType, SingletonPrompt,
9
- TextPrompt, TokensPrompt)
7
+from .data import (EncoderDecoderInputs, ExplicitEncoderDecoderPrompt,
8
+ ProcessorInputs, PromptType, SingletonPrompt, TextPrompt,
9
+ TokensPrompt)
10
11
12
class ParsedText(TypedDict):
@@ -98,12 +98,15 @@ def parse_singleton_prompt(
98
raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
99
100
101
+def is_token_prompt(prompt: PromptType) -> TypeIs[TokensPrompt]:
102
+ return isinstance(prompt, dict) and "prompt_token_ids" in prompt
103
+
104
+
105
def is_explicit_encoder_decoder_prompt(
106
prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
107
return isinstance(prompt, dict) and "encoder_prompt" in prompt
108
109
110
def is_encoder_decoder_inputs(
107
- inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
108
-) -> TypeIs[EncoderDecoderInputs]:
109
- return "encoder_prompt_token_ids" in inputs
111
+ inputs: ProcessorInputs) -> TypeIs[EncoderDecoderInputs]:
112
+ return "encoder" in inputs and "decoder" in inputs
vllm/inputs/preprocess.py
+136
-133
@@ -1,5 +1,5 @@
1
import asyncio
2
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
2
+from typing import List, Optional
3
4
from typing_extensions import assert_never
5
@@ -10,22 +10,12 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
10
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
from vllm.utils import print_warning_once
12
13
-from .data import (DecoderOnlyInputs, EncoderDecoderInputs, PromptType,
14
- SingletonPrompt)
13
+from .data import (DecoderOnlyInputs, EncoderDecoderInputs, ProcessorInputs,
14
+ PromptType, SingletonInputs, SingletonPrompt, token_inputs)
15
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
16
17
-if TYPE_CHECKING:
18
- from vllm.multimodal import MultiModalDataDict
19
-
17
logger = init_logger(__name__)
18
22
-PromptComponents = Tuple[Optional[str], List[int],
23
- Optional["MultiModalDataDict"], Optional[Dict[str,
24
- Any]]]
25
-DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
26
- Optional["MultiModalDataDict"],
27
- Optional[Dict[str, Any]]]
28
-
19
20
class InputPreprocessor:
21
@@ -115,7 +105,7 @@ class InputPreprocessor:
105
"default" decoder prompt be <BOS>.
106
107
However, it is possible that in the future
118
- other models may have different or more
108
+ other models may have different or more
109
complex logic for the default decoder prompt.
110
This motivates having a special helper method
111
for default decoder prompts.
@@ -132,7 +122,6 @@ class InputPreprocessor:
122
def _prepare_decoder_input_ids_for_generation(
123
self,
124
decoder_input_ids: Optional[List[int]],
135
- force_bos: bool = True,
125
) -> List[int]:
126
"""
127
Prepares `decoder_input_ids` for generation with encoder-decoder models.
@@ -162,8 +151,8 @@ class InputPreprocessor:
151
# use decoder_start_token_id as decoder_input_ids
152
decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
153
165
- if force_bos and (len(decoder_input_ids) == 0
166
- or decoder_input_ids[0] != decoder_start_token_id):
154
+ if (len(decoder_input_ids) == 0
155
+ or decoder_input_ids[0] != decoder_start_token_id):
156
decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
157
158
return decoder_input_ids
@@ -209,12 +198,12 @@ class InputPreprocessor:
198
prompt=prompt,
199
lora_request=lora_request)
200
212
- def _extract_prompt_components(
201
+ def _prompt_to_llm_inputs(
202
self,
203
prompt: SingletonPrompt,
204
request_id: str,
205
lora_request: Optional[LoRARequest] = None,
217
- ) -> PromptComponents:
206
+ ) -> SingletonInputs:
207
'''
208
Extract the components of any single encoder or decoder input prompt.
209
@@ -241,34 +230,52 @@ class InputPreprocessor:
230
request_id=request_id,
231
lora_request=lora_request,
232
)
244
- multi_modal_data = None
245
- mm_processor_kwargs = None
246
- elif parsed["type"] == "tokens":
247
- prompt_text = None
248
- prompt_token_ids = parsed["content"]["prompt_token_ids"]
249
- multi_modal_data = parsed["content"].get("multi_modal_data")
250
- mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
251
- elif parsed["type"] == "text":
252
- prompt_text = parsed["content"]["prompt"]
233
+
234
+ return token_inputs(
235
+ prompt=prompt_text,
236
+ prompt_token_ids=prompt_token_ids,
237
+ )
238
+
239
+ if parsed["type"] == "tokens":
240
+ tokens_content = parsed["content"]
241
+
242
+ prompt_token_ids = tokens_content["prompt_token_ids"]
243
+ multi_modal_data = tokens_content.get("multi_modal_data")
244
+ mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
245
+
246
+ return token_inputs(
247
+ prompt_token_ids=prompt_token_ids,
248
+ multi_modal_data=multi_modal_data,
249
+ mm_processor_kwargs=mm_processor_kwargs,
250
+ )
251
+
252
+ if parsed["type"] == "text":
253
+ text_content = parsed["content"]
254
+
255
+ prompt_text = text_content["prompt"]
256
prompt_token_ids = self._tokenize_prompt(
257
prompt_text,
258
request_id=request_id,
259
lora_request=lora_request,
260
)
258
- multi_modal_data = parsed["content"].get("multi_modal_data")
259
- mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
260
- else:
261
- assert_never(parsed)
261
+ multi_modal_data = text_content.get("multi_modal_data")
262
+ mm_processor_kwargs = text_content.get("mm_processor_kwargs")
263
+
264
+ return token_inputs(
265
+ prompt=prompt_text,
266
+ prompt_token_ids=prompt_token_ids,
267
+ multi_modal_data=multi_modal_data,
268
+ mm_processor_kwargs=mm_processor_kwargs,
269
+ )
270
263
- return (prompt_text, prompt_token_ids, multi_modal_data,
264
- mm_processor_kwargs)
271
+ assert_never(parsed)
272
266
- async def _extract_prompt_components_async(
273
+ async def _prompt_to_llm_inputs_async(
274
self,
275
prompt: SingletonPrompt,
276
request_id: str,
277
lora_request: Optional[LoRARequest] = None,
271
- ) -> PromptComponents:
278
+ ) -> SingletonInputs:
279
"""Async version of :meth:`_extract_prompt_components`."""
280
parsed = parse_singleton_prompt(prompt)
281
@@ -279,59 +286,74 @@ class InputPreprocessor:
286
request_id=request_id,
287
lora_request=lora_request,
288
)
282
- multi_modal_data = None
283
- mm_processor_kwargs = None
284
- elif parsed["type"] == "tokens":
285
- prompt_text = None
286
- prompt_token_ids = parsed["content"]["prompt_token_ids"]
287
- multi_modal_data = parsed["content"].get("multi_modal_data")
288
- mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
289
- elif parsed["type"] == "text":
290
- prompt_text = parsed["content"]["prompt"]
289
+
290
+ return token_inputs(
291
+ prompt=prompt_text,
292
+ prompt_token_ids=prompt_token_ids,
293
+ )
294
+
295
+ if parsed["type"] == "tokens":
296
+ tokens_content = parsed["content"]
297
+
298
+ prompt_token_ids = tokens_content["prompt_token_ids"]
299
+ multi_modal_data = tokens_content.get("multi_modal_data")
300
+ mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
301
+
302
+ return token_inputs(
303
+ prompt_token_ids=prompt_token_ids,
304
+ multi_modal_data=multi_modal_data,
305
+ mm_processor_kwargs=mm_processor_kwargs,
306
+ )
307
+
308
+ if parsed["type"] == "text":
309
+ text_content = parsed["content"]
310
+
311
+ prompt_text = text_content["prompt"]
312
prompt_token_ids = await self._tokenize_prompt_async(
313
prompt_text,
314
request_id=request_id,
315
lora_request=lora_request,
316
)
296
- multi_modal_data = parsed["content"].get("multi_modal_data")
297
- mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
298
- else:
299
- assert_never(parsed)
317
+ multi_modal_data = text_content.get("multi_modal_data")
318
+ mm_processor_kwargs = text_content.get("mm_processor_kwargs")
319
+
320
+ return token_inputs(
321
+ prompt=prompt_text,
322
+ prompt_token_ids=prompt_token_ids,
323
+ multi_modal_data=multi_modal_data,
324
+ mm_processor_kwargs=mm_processor_kwargs,
325
+ )
326
301
- return (prompt_text, prompt_token_ids, multi_modal_data,
302
- mm_processor_kwargs)
327
+ assert_never(parsed)
328
329
def _build_enc_dec_llm_inputs(
330
self,
306
- encoder_comps: PromptComponents,
307
- decoder_comps: DecoderPromptComponents,
308
- mm_processor_kwargs: Dict[str, Any],
331
+ encoder_inputs: SingletonInputs,
332
+ decoder_inputs: Optional[SingletonInputs],
333
) -> EncoderDecoderInputs:
310
- encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311
- decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312
-
313
- # Reminder: Please update docs/source/serving/compatibility_matrix.rst
314
- # If the feature combo become valid
315
- if decoder_mm_data is not None:
316
- raise ValueError(
317
- "Multi-modality decoder inputs of encoder-decoder models are "
318
- "not supported yet")
319
-
320
- # For Multi-Modal models (e.g., mllama), the text input can be
321
- # <|image|><|begin_of_text|>hello world. And we should not add
322
- # another <|begin_of_text|> to the beginning.
323
- decoder_prompt_ids = (self._prepare_decoder_input_ids_for_generation(
324
- decoder_prompt_ids,
325
- force_bos=(encoder_mm_data is None and decoder_mm_data is None)))
334
+ if encoder_inputs["type"] == "token":
335
+ pass
336
+ else:
337
+ assert_never(encoder_inputs)
338
+
339
+ if decoder_inputs is None:
340
+ dec_token_ids = self._prepare_decoder_input_ids_for_generation(
341
+ None)
342
+ decoder_inputs = token_inputs(dec_token_ids)
343
+ elif decoder_inputs["type"] == "token":
344
+ dec_token_ids = self._prepare_decoder_input_ids_for_generation(
345
+ decoder_inputs["prompt_token_ids"])
346
+ decoder_inputs["prompt_token_ids"] = dec_token_ids
347
+
348
+ if "multi_modal_data" in decoder_inputs:
349
+ raise ValueError("Multi-modal decoder inputs of encoder-"
350
+ "decoder models are not supported yet")
351
+ else:
352
+ assert_never(encoder_inputs)
353
354
return EncoderDecoderInputs(
328
- prompt_token_ids=decoder_prompt_ids,
329
- prompt=decoder_prompt,
330
- multi_modal_data=decoder_mm_data,
331
- mm_processor_kwargs=mm_processor_kwargs,
332
- encoder_prompt_token_ids=encoder_prompt_ids,
333
- encoder_prompt=encoder_prompt,
334
- encoder_multi_modal_data=encoder_mm_data,
355
+ encoder=encoder_inputs,
356
+ decoder=decoder_inputs,
357
)
358
359
def _process_encoder_decoder_prompt(
@@ -341,8 +363,7 @@ class InputPreprocessor:
363
) -> EncoderDecoderInputs:
364
'''
365
For encoder/decoder models only:
344
- Process an input prompt into an
345
- :class:`EncoderDecoderInputs` instance.
366
+ Process an input prompt into an :class:`EncoderDecoderInputs` instance.
367
368
There are two types of input prompts:
369
singleton prompts which carry only the
@@ -361,7 +382,7 @@ class InputPreprocessor:
382
have any possible singleton type; thus this
383
method relies on helper functions to obtain
384
token ids for the sub-prompts.
364
-
385
+
386
Arguments:
387
388
* prompt: an input prompt
@@ -372,40 +393,31 @@ class InputPreprocessor:
393
* :class:`EncoderDecoderInputs` instance
394
'''
395
375
- encoder_comps: PromptComponents
376
- decoder_comps: DecoderPromptComponents
396
+ encoder_inputs: SingletonInputs
397
+ decoder_inputs: Optional[SingletonInputs]
398
399
if is_explicit_encoder_decoder_prompt(prompt):
379
- encoder_comps = self._extract_prompt_components(
400
+ encoder_inputs = self._prompt_to_llm_inputs(
401
prompt["encoder_prompt"],
402
request_id=request_id,
403
)
404
405
if (decoder_input := prompt["decoder_prompt"]) is None:
385
- decoder_comps = None, None, None, None
406
+ decoder_inputs = None
407
else:
387
- decoder_comps = self._extract_prompt_components(
408
+ decoder_inputs = self._prompt_to_llm_inputs(
409
decoder_input,
410
request_id=request_id,
411
)
391
- # Handle this carefully in case it was directly initialized by user
392
- mm_processor_kwargs = prompt.get("mm_processor_kwargs", {})
412
else:
394
- encoder_comps = self._extract_prompt_components(
413
+ encoder_inputs = self._prompt_to_llm_inputs(
414
prompt,
415
request_id=request_id,
416
)
398
- # If there are no decoder components, we assume the
399
- # mm_processor_kwargs are in the encoder prompt
400
- mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
401
- -1] is not None else {}
402
- decoder_comps = None, None, None, None
403
-
404
- return self._build_enc_dec_llm_inputs(
405
- encoder_comps,
406
- decoder_comps,
407
- mm_processor_kwargs,
408
- )
417
+
418
+ decoder_inputs = None
419
+
420
+ return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
421
422
async def _process_encoder_decoder_prompt_async(
423
self,
@@ -413,59 +425,50 @@ class InputPreprocessor:
425
request_id: str,
426
) -> EncoderDecoderInputs:
427
"""Async version of :meth:`_process_encoder_decoder_prompt`."""
416
- encoder_comps: PromptComponents
417
- decoder_comps: DecoderPromptComponents
428
+ encoder_inputs: SingletonInputs
429
+ decoder_inputs: Optional[SingletonInputs]
430
431
if is_explicit_encoder_decoder_prompt(prompt):
420
- encoder_task = self._extract_prompt_components_async(
432
+ encoder_task = self._prompt_to_llm_inputs_async(
433
prompt["encoder_prompt"],
434
request_id=request_id,
435
)
436
437
if (decoder_input := prompt["decoder_prompt"]) is None:
426
- encoder_comps = await encoder_task
427
- decoder_comps = None, None, None, None
438
+ encoder_inputs = await encoder_task
439
+ decoder_inputs = None
440
else:
429
- decoder_task = self._extract_prompt_components_async(
441
+ decoder_task = self._prompt_to_llm_inputs_async(
442
decoder_input,
443
request_id=request_id,
444
)
445
434
- encoder_comps, decoder_comps = await asyncio.gather(
446
+ encoder_inputs, decoder_inputs = await asyncio.gather(
447
encoder_task, decoder_task)
436
- mm_processor_kwargs = prompt["mm_processor_kwargs"]
448
else:
438
- encoder_comps = await self._extract_prompt_components_async(
449
+ encoder_inputs = await self._prompt_to_llm_inputs_async(
450
prompt,
451
request_id=request_id,
452
)
442
- # If there are no decoder components, we assume the
443
- # mm_processor_kwargs are in the encoder prompt
444
- mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
445
- -1] is not None else {}
446
- decoder_comps = None, None, None, None
447
-
448
- return self._build_enc_dec_llm_inputs(
449
- encoder_comps,
450
- decoder_comps,
451
- mm_processor_kwargs,
452
- )
453
+
454
+ decoder_inputs = None
455
+
456
+ return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
457
458
def _build_decoder_only_llm_inputs(
459
self,
456
- prompt_comps: PromptComponents,
460
+ prompt_inputs: DecoderOnlyInputs,
461
prompt_adapter_request: Optional[PromptAdapterRequest],
462
) -> DecoderOnlyInputs:
459
- (prompt, prompt_token_ids, multi_modal_data,
460
- mm_processor_kwargs) = prompt_comps
461
-
462
- prompt_token_ids = self._apply_prompt_adapter(
463
- prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
463
+ if prompt_inputs["type"] == "token":
464
+ prompt_inputs["prompt_token_ids"] = self._apply_prompt_adapter(
465
+ prompt_inputs["prompt_token_ids"],
466
+ prompt_adapter_request=prompt_adapter_request,
467
+ )
468
+ else:
469
+ assert_never(prompt_inputs)
470
465
- return DecoderOnlyInputs(prompt_token_ids=prompt_token_ids,
466
- prompt=prompt,
467
- multi_modal_data=multi_modal_data,
468
- mm_processor_kwargs=mm_processor_kwargs)
471
+ return prompt_inputs
472
473
def _process_decoder_only_prompt(
474
self,
@@ -490,7 +493,7 @@ class InputPreprocessor:
493
* :class:`DecoderOnlyInputs` instance
494
'''
495
493
- prompt_comps = self._extract_prompt_components(
496
+ prompt_comps = self._prompt_to_llm_inputs(
497
prompt,
498
request_id=request_id,
499
lora_request=lora_request,
@@ -509,7 +512,7 @@ class InputPreprocessor:
512
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
513
) -> DecoderOnlyInputs:
514
"""Async version of :meth:`_process_decoder_only_prompt`."""
512
- prompt_comps = await self._extract_prompt_components_async(
515
+ prompt_comps = await self._prompt_to_llm_inputs_async(
516
prompt,
517
request_id=request_id,
518
lora_request=lora_request,
@@ -526,7 +529,7 @@ class InputPreprocessor:
529
request_id: str,
530
lora_request: Optional[LoRARequest] = None,
531
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
529
- ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
532
+ ) -> ProcessorInputs:
533
"""Preprocess the input prompt."""
534
if self.is_encoder_decoder_model():
535
# Encoder-decoder model requires special mapping of
@@ -554,7 +557,7 @@ class InputPreprocessor:
557
request_id: str,
558
lora_request: Optional[LoRARequest] = None,
559
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
557
- ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
560
+ ) -> ProcessorInputs:
561
"""Async version of :meth:`preprocess`."""
562
if self.is_encoder_decoder_model():
563
# Encoder-decoder model requires special mapping of
vllm/inputs/registry.py
+7
-7
@@ -2,7 +2,7 @@ import functools
2
from collections import UserDict
3
from dataclasses import dataclass
4
from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5
- Optional, Protocol, Type)
5
+ Optional, Protocol, Type, cast)
6
7
from torch import nn
8
from transformers import PretrainedConfig
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
12
from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
13
resolve_mm_processor_kwargs)
14
15
-from .data import DecoderOnlyInputs
15
+from .data import ProcessorInputs
16
17
if TYPE_CHECKING:
18
from vllm.config import ModelConfig
@@ -109,7 +109,7 @@ class _MultiModalCounts(UserDict):
109
raise KeyError(msg) from exc
110
111
112
-InputProcessor = Callable[[InputContext, DecoderOnlyInputs], DecoderOnlyInputs]
112
+InputProcessor = Callable[[InputContext, ProcessorInputs], ProcessorInputs]
113
"""Preprocess the inputs to the model."""
114
115
@@ -254,8 +254,8 @@ class InputRegistry:
254
def _default_input_processor(
255
self,
256
ctx: InputContext,
257
- inputs: DecoderOnlyInputs,
258
- ) -> DecoderOnlyInputs:
257
+ inputs: ProcessorInputs,
258
+ ) -> ProcessorInputs:
259
"""The default input processor is a no-op."""
260
return inputs
261
@@ -288,7 +288,7 @@ class InputRegistry:
288
.get(model_cls, self._default_input_processor)
289
290
def process_input(self, model_config: "ModelConfig",
291
- inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
291
+ inputs: ProcessorInputs) -> ProcessorInputs:
292
"""
293
Apply an input processor to an instance of model inputs.
294
@@ -308,7 +308,7 @@ class InputRegistry:
308
# If it's empty, it'll fall back to the default kwarg values
309
mm_processor_kwargs = resolve_mm_processor_kwargs(
310
model_config.mm_processor_kwargs,
311
- inputs.get("mm_processor_kwargs"),
311
+ cast(Dict[str, Any], inputs.get("mm_processor_kwargs")),
312
processor,
313
)
314
vllm/model_executor/models/mllama.py
+67
-29
@@ -36,8 +36,8 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
36
from vllm.attention.ops.paged_attn import PagedAttention
37
from vllm.config import CacheConfig, MultiModalConfig
38
from vllm.distributed import get_tensor_model_parallel_world_size
39
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
40
- EncoderDecoderInputs, InputContext)
39
+from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
40
+ InputContext, TokenInputs, token_inputs)
41
from vllm.logger import init_logger
42
from vllm.model_executor.layers.layernorm import RMSNorm
43
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -52,6 +52,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
from vllm.multimodal import MULTIMODAL_REGISTRY
54
from vllm.sequence import SequenceData
55
+from vllm.utils import is_list_of
56
57
from .clip import CLIPMLP
58
from .interfaces import SupportsMultiModal
@@ -86,41 +87,58 @@ def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
87
return num_images
88
89
89
-def input_processor_for_mllama(ctx: InputContext,
90
- inputs: Union[DecoderOnlyInputs,
91
- EncoderDecoderInputs]):
92
- # move encoder_prompt to prompt
93
- if inputs.get("prompt") is None:
94
- inputs["prompt"] = inputs["encoder_prompt"]
95
- inputs["prompt_token_ids"] = inputs["encoder_prompt_token_ids"]
90
+def input_processor_for_mllama(
91
+ ctx: InputContext,
92
+ inputs: EncoderDecoderInputs,
93
+) -> EncoderDecoderInputs:
94
+ # Example input to processor:
95
+ # {
96
+ # 'encoder': {
97
+ # 'type': 'token',
98
+ # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
99
+ # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
100
+ # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
101
+ # },
102
+ # 'decoder': {
103
+ # 'type': 'token',
104
+ # 'prompt_token_ids': [128000],
105
+ # },
106
+ # }
107
+
108
+ # move encoder prompt to decoder
109
+ dec_inputs = TokenInputs(**inputs["encoder"])
110
+
111
+ multi_modal_data = dec_inputs.get("multi_modal_data")
112
+ if multi_modal_data is None or "image" not in multi_modal_data:
113
+ # text-only
114
+ return EncoderDecoderInputs(
115
+ encoder=token_inputs([]),
116
+ decoder=dec_inputs,
117
+ )
118
97
- # process multi-modal data
98
- multi_modal_data = inputs.get("encoder_multi_modal_data")
119
+ image_data = multi_modal_data["image"]
120
+ if isinstance(image_data, Image.Image):
121
+ image_data = [image_data]
122
100
- if multi_modal_data is None or "image" not in multi_modal_data \
101
- or multi_modal_data["image"] is None:
102
- # text-only
103
- inputs["encoder_prompt"] = ""
104
- inputs["encoder_prompt_token_ids"] = []
105
- inputs["encoder_multi_modal_data"] = {}
106
- return inputs
123
+ assert is_list_of(image_data, Image.Image)
124
108
- if isinstance(multi_modal_data['image'], Image.Image):
109
- multi_modal_data['image'] = [multi_modal_data['image']]
125
# Since only the last group of consecutive images
126
# are attended by the decoded tokens, we only need to
127
# get the number of tiles for those images.
128
num_decode_images = _get_num_image_in_last_group(
114
- inputs["prompt_token_ids"])
129
+ dec_inputs["prompt_token_ids"])
130
+
131
hf_config = ctx.model_config.hf_config
132
+ vision_config = hf_config.vision_config
133
+
134
num_tiles = 0
117
- for image in multi_modal_data["image"][::-1]:
135
+ for image in image_data[::-1]:
136
width, height = image.size
119
- tile_size = hf_config.vision_config.image_size
137
+ tile_size = vision_config.image_size
138
canvas_height, canvas_width = get_optimal_tiled_canvas(
139
image_height=height,
140
image_width=width,
123
- max_image_tiles=hf_config.vision_config.max_num_tiles,
141
+ max_image_tiles=vision_config.max_num_tiles,
142
tile_size=tile_size,
143
)
144
num_tiles_height = canvas_height // tile_size
@@ -133,14 +151,34 @@ def input_processor_for_mllama(ctx: InputContext,
151
# Set encoder prompt length based on the number of tiles.
152
# This tells the block manager to allocate correct number
153
# of slots for encoder tokens.
136
- assert hf_config.vision_config.image_size % 14 == 0, \
154
+ assert vision_config.image_size % 14 == 0, \
155
"chunk size should be multiple of 14"
138
- token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
156
+ token_per_chunk = (vision_config.image_size // 14)**2 + 1
157
num_tokens = num_tiles * token_per_chunk
140
- inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
141
- inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID] * num_tokens
158
143
- return inputs
159
+ # Example output from processor:
160
+ # {
161
+ # 'encoder': {
162
+ # 'type': 'token',
163
+ # 'prompt_token_ids': [128256, 128256, ..., 128256],
164
+ # 'prompt': '<|image|><|image|>...<|image|>',
165
+ # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
166
+ # },
167
+ # 'decoder': {
168
+ # 'type': 'token',
169
+ # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
170
+ # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
171
+ # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
172
+ # },
173
+ # }
174
+ return EncoderDecoderInputs(
175
+ encoder=token_inputs(
176
+ prompt_token_ids=[MLLAMA_IMAGE_TOKEN_ID] * num_tokens,
177
+ prompt=MLLAMA_IMAGE_TOKEN * num_tokens,
178
+ multi_modal_data=multi_modal_data,
179
+ ),
180
+ decoder=dec_inputs,
181
+ )
182
183
184
def get_max_mllama_image_tokens(ctx: InputContext) -> int:
vllm/model_executor/models/registry.py
+5
@@ -343,6 +343,11 @@ class _ModelRegistry:
343
def _raise_for_unsupported(self, architectures: List[str]):
344
all_supported_archs = self.get_supported_archs()
345
346
+ if any(arch in all_supported_archs for arch in architectures):
347
+ raise ValueError(
348
+ f"Model architectures {architectures} failed "
349
+ "to be inspected. Please check the logs for more details.")
350
+
351
raise ValueError(
352
f"Model architectures {architectures} are not supported for now. "
353
f"Supported architectures: {all_supported_archs}")
vllm/sequence.py
+44
-69
@@ -9,12 +9,12 @@ from functools import cached_property, reduce
9
from typing import (TYPE_CHECKING, Any, Callable, DefaultDict, Dict, List,
10
Mapping, Optional)
11
from typing import Sequence as GenericSequence
12
-from typing import Set, Tuple, Union, cast
12
+from typing import Set, Tuple, Union
13
14
import msgspec
15
import torch
16
+from typing_extensions import assert_never
17
17
-from vllm.inputs.parse import is_encoder_decoder_inputs
18
from vllm.lora.request import LoRARequest
19
from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
20
from vllm.pooling_params import PoolingParams
@@ -379,15 +379,10 @@ class SequenceData(msgspec.Struct,
379
380
class Sequence:
381
"""Stores the data, status, and block information of a sequence.
382
-
383
- The sequence is constructed from the :code:`SingletonInputs` instance
384
- passed in through the :code:`inputs` constructor argument.
385
-
386
- For encoder/decoder models, SingletonInputs encapsulates both a
387
- decoder and encoder prompt, creating an ambiguity about which
388
- prompt to construct the sequence from. The `from_decoder_prompt`
389
- constructor argument signals whether to construct the Sequence
390
- from the SingletonInputs decoder prompt, or encoder prompt.
382
+
383
+ The sequence is constructed from the :data:`DecoderOnlyInputs`
384
+ (for decoder-only) or :data:`EncoderDecoderInputs` (for encoder-decoder)
385
+ instance passed in through the :code:`inputs` constructor argument.
386
387
Args:
388
seq_id: The ID of the sequence.
@@ -397,10 +392,6 @@ class Sequence:
392
eos_token_id: The end-of-sequence (EOS) token id recognized by this LLM.
393
lora_request: LoRA request.
394
prompt_adapter_request: Prompt Adapter request.
400
- from_decoder_prompt: Construct Sequence from SingletonInputs decoder
401
- prompt (True) or encoder prompt (False.) Must be
402
- True for decoder-only model.
403
-
395
"""
396
397
def __init__(
@@ -411,7 +402,6 @@ class Sequence:
402
eos_token_id: Optional[int] = None,
403
lora_request: Optional[LoRARequest] = None,
404
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
414
- from_decoder_prompt: bool = True,
405
) -> None:
406
self.seq_id = seq_id
407
self.inputs = inputs
@@ -419,33 +409,6 @@ class Sequence:
409
self.eos_token_id = eos_token_id
410
self.lora_request = lora_request
411
self.prompt_adapter_request = prompt_adapter_request
422
- self.from_decoder_prompt = from_decoder_prompt
423
-
424
- # For decoder-only models, a Sequence is constructed
425
- # from an DecoderOnlyInputs instance (the `inputs` arg.)
426
- #
427
- # For encoder/decoder models the same `inputs`
428
- # instance could be utilized to construct either an
429
- # encoder sequence or a decoder sequence, because
430
- # `DecoderOnlyInputs` has both decoder- and encoder-oriented
431
- # member variables (i.e. it encapsulates both an encoder
432
- # and a decoder prompt.) The decision of which type of sequence
433
- # to generate is determined by the `from_decoder_prompt` argument.
434
- #
435
- # When constructing a encoder sequence
436
- # (`from_decoder_prompt` False) it matters that
437
- # the `DecoderOnlyInputs` instance stored in `inputs` is valid
438
- # in the sense that its encoder-related member variables are
439
- # populated; below, an exception is raised if this is
440
- # not the case.
441
- #
442
- # When constructing a decoder sequence (`from_decoder_prompt` True)
443
- # it does not matter whether `inputs` has its encoder-related
444
- # member variables populated.
445
- if not (from_decoder_prompt or is_encoder_decoder_inputs(inputs)):
446
- raise ValueError("Cannot extract encoder input prompt from "
447
- f"invalid input {inputs}; did you forget the "
448
- "encoder input prompt fields?")
412
413
self.data = SequenceData.from_seqs(self.prompt_token_ids)
414
self.output_logprobs: SampleLogprobs = []
@@ -470,45 +433,57 @@ class Sequence:
433
434
@cached_property
435
def prompt(self) -> Optional[str]:
473
- # Select decoder or encoder input prompt str, as appropriate
474
- prompt_key: str = ("prompt"
475
- if self.from_decoder_prompt else "encoder_prompt")
436
+ inputs = self.inputs
437
477
- return cast(Optional[str], self.inputs.get(prompt_key))
438
+ if inputs["type"] == "token":
439
+ return inputs.get("prompt")
440
+
441
+ assert_never(inputs)
442
443
@cached_property
444
def prompt_token_ids(self) -> List[int]:
481
- # Select decoder or encoder input prompt token ids, as appropriate
482
- prompt_token_ids_key: str = ("prompt_token_ids"
483
- if self.from_decoder_prompt else
484
- "encoder_prompt_token_ids")
445
+ inputs = self.inputs
446
486
- # Cache computed prompt token ids
487
- return cast(List[int], self.inputs.get(prompt_token_ids_key))
447
+ if inputs["type"] == "token":
448
+ return inputs.get("prompt_token_ids", [])
449
489
- @property
490
- def multi_modal_data(self) -> MultiModalDataDict:
450
+ assert_never(inputs)
451
+
452
+ @cached_property
453
+ def prompt_embeds(self) -> Optional[torch.Tensor]:
454
inputs = self.inputs
455
493
- if (inputs.get("multi_modal_data")
494
- and inputs.get("encoder_multi_modal_data")):
495
- raise ValueError(
496
- "Multi-modal data in both encoder and decoder is not supported."
497
- )
456
+ if inputs["type"] == "token":
457
+ return None
458
499
- return cast(
500
- MultiModalDataDict,
501
- (inputs.get("multi_modal_data")
502
- or inputs.get("encoder_multi_modal_data") or {}),
503
- )
459
+ assert_never(inputs)
460
+
461
+ @cached_property
462
+ def multi_modal_data(self) -> "MultiModalDataDict":
463
+ inputs = self.inputs
464
+
465
+ if inputs["type"] == "token":
466
+ return inputs.get("multi_modal_data", {})
467
+
468
+ assert_never(inputs)
469
+
470
+ @cached_property
471
+ def mm_processor_kwargs(self) -> Dict[str, Any]:
472
+ inputs = self.inputs
473
+
474
+ if inputs["type"] == "token":
475
+ return inputs.get("mm_processor_kwargs", {})
476
+
477
+ assert_never(inputs)
478
479
@property
480
def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
507
- return self.inputs.get("multi_modal_placeholders") or {}
481
+ inputs = self.inputs
482
509
- @property
510
- def mm_processor_kwargs(self) -> Dict[str, Any]:
511
- return self.inputs.get("mm_processor_kwargs") or {}
483
+ if inputs["type"] == "token":
484
+ return inputs.get("multi_modal_placeholders", {})
485
+
486
+ assert_never(inputs)
487
488
@property
489
def lora_int_id(self) -> int: