@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bbc3619dc8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Core] Make encoder-decoder inputs a nested structure to be more composable (#9604)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 5, 2024 at 10:07 UTC bbc3619dc806b25fd5e14eef90819052ab76e1c6
14 files changed +372 -349
tests/core/utils.py
+26 -31
@@ -4,6 +4,7 @@ from typing import Sequence as GenericSequence
4 from typing import Tuple
5
6 from vllm import SamplingParams
7 +from vllm.inputs import EncoderDecoderInputs, token_inputs
8 from vllm.lora.request import LoRARequest
9 from vllm.sequence import Logprob, Sequence, SequenceGroup
10
@@ -27,10 +28,7 @@ def create_dummy_prompt(
28 prompt_tokens = list(range(prompt_length))
29 prompt_str = " ".join([str(t) for t in prompt_tokens])
30 prompt = Sequence(int(request_id),
30 - inputs={
31 - "prompt": prompt_str,
32 - "prompt_token_ids": prompt_tokens,
33 - },
31 + inputs=token_inputs(prompt_tokens, prompt=prompt_str),
32 block_size=block_size)
33 seq_group = SequenceGroup(request_id=request_id,
34 seqs=[prompt],
@@ -63,23 +61,21 @@ def create_dummy_prompt_encoder_decoder(
61 encoder_prompt_tokens = list(reversed(list(range(encoder_prompt_length))))
62 encoder_prompt_str = " ".join([str(t) for t in encoder_prompt_tokens])
63
66 - inputs = {
67 - "prompt": decoder_prompt_str,
68 - "prompt_token_ids": decoder_prompt_tokens,
69 - "encoder_prompt": encoder_prompt_str,
70 - "encoder_prompt_token_ids": encoder_prompt_tokens,
71 - "multi_modal_data": None,
64 + inputs: EncoderDecoderInputs = {
65 + "decoder": token_inputs(decoder_prompt_tokens,
66 + prompt=decoder_prompt_str),
67 + "encoder": token_inputs(encoder_prompt_tokens,
68 + prompt=encoder_prompt_str),
69 }
70
71 decoder_prompt = Sequence(int(request_id),
75 - inputs=inputs,
76 - block_size=block_size,
77 - from_decoder_prompt=True)
72 + inputs=inputs["decoder"],
73 + block_size=block_size)
74
75 encoder_prompt = Sequence(int(request_id),
80 - inputs=inputs,
81 - block_size=block_size,
82 - from_decoder_prompt=False)
76 + inputs=inputs["encoder"],
77 + block_size=block_size)
78 +
79 seq_group = SequenceGroup(request_id=request_id,
80 seqs=[decoder_prompt],
81 sampling_params=SamplingParams(best_of=best_of),
@@ -108,7 +104,7 @@ def create_seq_group(
104 for seq_id_offset, output_len in enumerate(seq_output_lens):
105 seq = Sequence(
106 seq_id=seq_id_start + seq_id_offset,
111 - inputs={"prompt_token_ids": prompt_token_ids},
107 + inputs=token_inputs(prompt_token_ids),
108 block_size=16,
109 )
110
@@ -143,21 +139,19 @@ def create_seq_group_encoder_decoder(
139
140 prompt_token_ids = [0] * seq_prompt_len
141
146 - inputs = {
147 - "prompt": "",
148 - "prompt_token_ids": prompt_token_ids,
149 - "encoder_prompt": "",
150 - "encoder_prompt_token_ids": prompt_token_ids,
151 - "multi_modal_data": None,
142 + inputs: EncoderDecoderInputs = {
143 + "decoder": token_inputs(prompt_token_ids),
144 + "encoder": token_inputs(prompt_token_ids),
145 }
146
147 seqs = []
148 for seq_id_offset, output_len in enumerate(seq_output_lens):
149 # Construct decoder input sequences
157 - seq = Sequence(seq_id=seq_id_start + seq_id_offset,
158 - inputs=inputs,
159 - block_size=16,
160 - from_decoder_prompt=True)
150 + seq = Sequence(
151 + seq_id=seq_id_start + seq_id_offset,
152 + inputs=inputs["decoder"],
153 + block_size=16,
154 + )
155
156 for i in range(output_len):
157 seq.append_token_id(
@@ -167,10 +161,11 @@ def create_seq_group_encoder_decoder(
161 seqs.append(seq)
162
163 # Encoder input sequence
170 - encoder_seq = Sequence(seq_id=seq_id_start + len(seq_output_lens),
171 - inputs=inputs,
172 - block_size=16,
173 - from_decoder_prompt=False)
164 + encoder_seq = Sequence(
165 + seq_id=seq_id_start + len(seq_output_lens),
166 + inputs=inputs["encoder"],
167 + block_size=16,
168 + )
169
170 return SequenceGroup(request_id=request_id,
171 seqs=seqs,
tests/engine/output_processor/test_stop_checker.py
+2 -1
@@ -4,6 +4,7 @@ import pytest
4 from transformers import PreTrainedTokenizer
5
6 from vllm.engine.output_processor.stop_checker import StopChecker
7 +from vllm.inputs import token_inputs
8 from vllm.sampling_params import SamplingParams
9 from vllm.sequence import Logprob, Sequence, SequenceStatus
10
@@ -15,7 +16,7 @@ def sequence_with_eos(text: str, eos_token: str,
16 """
17 seq = Sequence(
18 seq_id=0,
18 - inputs={"prompt_token_ids": []},
19 + inputs=token_inputs([]),
20 block_size=16,
21 eos_token_id=eos_token_id,
22 )
tests/test_cache_block_hashing.py
+3 -4
@@ -6,6 +6,7 @@ from typing import List, Optional
6
7 import pytest
8
9 +from vllm.inputs import token_inputs
10 from vllm.lora.request import LoRARequest
11 from vllm.sequence import Sequence
12 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -70,10 +71,8 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
71 hashes[-1].append([])
72 prompt_token_ids = tokenizer.encode(prompt)
73 seq = Sequence(seq_id,
73 - inputs={
74 - "prompt": prompt,
75 - "prompt_token_ids": prompt_token_ids,
76 - },
74 + inputs=token_inputs(prompt_token_ids,
75 + prompt=prompt),
76 block_size=block_size,
77 eos_token_id=tokenizer.tokenizer.eos_token_id,
78 lora_request=lora_request)
tests/tokenization/test_detokenize.py
+2 -4
@@ -3,6 +3,7 @@ from typing import Any, Dict, Generator, List, Optional
3 import pytest
4 from transformers import AutoTokenizer
5
6 +from vllm.inputs import token_inputs
7 from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
8 from vllm.transformers_utils.detokenizer import (Detokenizer,
9 detokenize_incrementally)
@@ -169,10 +170,7 @@ def create_sequence(prompt_token_ids=None):
170 prompt_token_ids = prompt_token_ids or [1]
171 return Sequence(
172 seq_id=0,
172 - inputs={
173 - "prompt": "<s>",
174 - "prompt_token_ids": prompt_token_ids,
175 - },
173 + inputs=token_inputs(prompt_token_ids, prompt="<s>"),
174 block_size=16,
175 )
176
vllm/engine/llm_engine.py
+24 -27
@@ -10,7 +10,7 @@ from typing import Sequence as GenericSequence
10 from typing import Set, Type, Union, cast, overload
11
12 import torch
13 -from typing_extensions import TypeIs, TypeVar
13 +from typing_extensions import TypeVar
14
15 import vllm.envs as envs
16 from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
@@ -29,9 +29,9 @@ from vllm.entrypoints.openai.logits_processors import (
29 from vllm.executor.executor_base import ExecutorBase
30 from vllm.executor.gpu_executor import GPUExecutor
31 from vllm.executor.ray_utils import initialize_ray_cluster
32 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
33 - EncoderDecoderInputs, InputRegistry, PromptType,
34 - TokensPrompt)
32 +from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
33 + PromptType)
34 +from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt
35 from vllm.inputs.preprocess import InputPreprocessor
36 from vllm.logger import init_logger
37 from vllm.logits_process import get_bad_words_logits_processors
@@ -638,7 +638,7 @@ class LLMEngine:
638 def _add_processed_request(
639 self,
640 request_id: str,
641 - processed_inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
641 + processed_inputs: ProcessorInputs,
642 params: Union[SamplingParams, PoolingParams],
643 arrival_time: float,
644 lora_request: Optional[LoRARequest],
@@ -669,18 +669,19 @@ class LLMEngine:
669 seq_id = next(self.seq_counter)
670 eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
671
672 - seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
672 + if is_encoder_decoder_inputs(processed_inputs):
673 + decoder_inputs = processed_inputs["decoder"]
674 + encoder_inputs = processed_inputs["encoder"]
675 + else:
676 + decoder_inputs = processed_inputs
677 + encoder_inputs = None
678 +
679 + seq = Sequence(seq_id, decoder_inputs, block_size, eos_token_id,
680 lora_request, prompt_adapter_request)
681
675 - encoder_seq = None
676 - if 'encoder_prompt_token_ids' in processed_inputs:
677 - encoder_seq = Sequence(seq_id,
678 - processed_inputs,
679 - block_size,
680 - eos_token_id,
681 - lora_request,
682 - prompt_adapter_request,
683 - from_decoder_prompt=False)
682 + encoder_seq = (None if encoder_inputs is None else Sequence(
683 + seq_id, encoder_inputs, block_size, eos_token_id, lora_request,
684 + prompt_adapter_request))
685
686 # Create a SequenceGroup based on SamplingParams or PoolingParams
687 if isinstance(params, SamplingParams):
@@ -874,7 +875,7 @@ class LLMEngine:
875 # This needs to happen before multimodal input pre-processing, which
876 # may add dummy <image> tokens that aren't part of the tokenizer's
877 # vocabulary.
877 - if self._is_token_prompt(prompt):
878 + if is_token_prompt(prompt):
879 prompt_ids = prompt["prompt_token_ids"]
880 if len(prompt_ids) == 0:
881 # Empty prompt check is handled later
@@ -884,10 +885,6 @@ class LLMEngine:
885 raise ValueError(
886 "Token id {} is out of vocabulary".format(max_input_id))
887
887 - @staticmethod
888 - def _is_token_prompt(prompt: PromptType) -> TypeIs[TokensPrompt]:
889 - return isinstance(prompt, dict) and "prompt_token_ids" in prompt
890 -
888 def _create_sequence_group_with_sampling(
889 self,
890 request_id: str,
@@ -1978,17 +1975,17 @@ class LLMEngine:
1975 def is_encoder_decoder_model(self):
1976 return self.input_preprocessor.is_encoder_decoder_model()
1977
1981 - def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1982 - EncoderDecoderInputs],
1978 + def _validate_model_inputs(self, inputs: ProcessorInputs,
1979 lora_request: Optional[LoRARequest]):
1984 - if self.model_config.is_multimodal_model:
1980 + if is_encoder_decoder_inputs(inputs):
1981 # For encoder-decoder multimodal models, the max_prompt_len
1982 # restricts the decoder prompt length
1987 - prompt_ids = inputs.get("prompt_token_ids")
1988 - elif self.is_encoder_decoder_model():
1989 - prompt_ids = inputs.get("encoder_prompt_token_ids")
1983 + prompt_inputs = inputs["decoder" if self.model_config.
1984 + is_multimodal_model else "encoder"]
1985 else:
1991 - prompt_ids = inputs.get("prompt_token_ids")
1986 + prompt_inputs = inputs
1987 +
1988 + prompt_ids = prompt_inputs.get("prompt_token_ids")
1989
1990 if prompt_ids is None or len(prompt_ids) == 0:
1991 raise ValueError("Prompt cannot be empty")
vllm/engine/protocol.py
+16 -7
@@ -1,11 +1,12 @@
1 import asyncio
2 from abc import ABC, abstractmethod
3 -from typing import AsyncGenerator, List, Mapping, Optional, Union
3 +from typing import AsyncGenerator, List, Mapping, Optional
4
5 from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
6 from vllm.config import DecodingConfig, ModelConfig
7 from vllm.core.scheduler import SchedulerOutputs
8 from vllm.inputs.data import PromptType, TokensPrompt
9 +from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
10 from vllm.inputs.preprocess import InputPreprocessor
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
@@ -60,7 +61,7 @@ class EngineClient(ABC):
61
62 async def beam_search(
63 self,
63 - prompt: Union[PromptType, List[int]],
64 + prompt: PromptType,
65 model_config: ModelConfig,
66 request_id: str,
67 params: BeamSearchParams,
@@ -76,11 +77,19 @@ class EngineClient(ABC):
77 tokenizer = await self.get_tokenizer()
78 input_preprocessor = InputPreprocessor(model_config, tokenizer)
79
79 - (prompt_text, prompt_token_ids, multi_modal_data,
80 - mm_processor_kwargs) = input_preprocessor._extract_prompt_components(
81 - prompt,
82 - request_id=request_id,
83 - )
80 + if is_explicit_encoder_decoder_prompt(prompt):
81 + raise NotImplementedError
82 + else:
83 + processed_inputs = input_preprocessor._prompt_to_llm_inputs(
84 + prompt,
85 + request_id=request_id,
86 + )
87 +
88 + prompt_token_ids = processed_inputs["prompt_token_ids"]
89 + prompt_text = processed_inputs.get("prompt")
90 + multi_modal_data = processed_inputs.get("multi_modal_data")
91 + mm_processor_kwargs = processed_inputs.get("mm_processor_kwargs")
92 +
93 tokenized_length = len(prompt_token_ids)
94
95 sort_beams_key = create_sort_beams_key_function(
vllm/inputs/__init__.py
+6 -5
@@ -1,8 +1,8 @@
1 from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
2 - ExplicitEncoderDecoderPrompt, PromptType, SingletonInputs,
3 - SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
4 - build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
5 - token_inputs, zip_enc_dec_prompts)
2 + ExplicitEncoderDecoderPrompt, ProcessorInputs, PromptType,
3 + SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
4 + TokensPrompt, build_explicit_enc_dec_prompt,
5 + to_enc_dec_tuple_list, token_inputs, zip_enc_dec_prompts)
6 from .registry import DummyData, InputContext, InputRegistry
7
8 INPUT_REGISTRY = InputRegistry()
@@ -22,9 +22,10 @@ __all__ = [
22 "ExplicitEncoderDecoderPrompt",
23 "TokenInputs",
24 "token_inputs",
25 - "SingletonInputs",
25 "DecoderOnlyInputs",
26 "EncoderDecoderInputs",
27 + "ProcessorInputs",
28 + "SingletonInputs",
29 "build_explicit_enc_dec_prompt",
30 "to_enc_dec_tuple_list",
31 "zip_enc_dec_prompts",
vllm/inputs/data.py
+25 -26
@@ -1,4 +1,4 @@
1 -from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
1 +from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List, Literal,
2 Optional, Tuple, Union, cast)
3
4 from typing_extensions import NotRequired, TypedDict, TypeVar
@@ -122,27 +122,30 @@ both decoder-only and encoder/decoder input types:
122
123 class TokenInputs(TypedDict):
124 """Represents token-based inputs."""
125 +
126 + type: Literal["token"]
127 + """The type of inputs."""
128 +
129 prompt_token_ids: List[int]
130 """The token IDs of the prompt."""
131
128 - prompt: NotRequired[Optional[str]]
132 + prompt: NotRequired[str]
133 """
134 The original prompt text corresponding to the token IDs, if available.
135 """
136
133 - multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
137 + multi_modal_data: NotRequired["MultiModalDataDict"]
138 """
139 Optional multi-modal data to pass to the model,
140 if the model supports it.
141 """
142
139 - multi_modal_placeholders: NotRequired[
140 - Optional["MultiModalPlaceholderDict"]]
143 + multi_modal_placeholders: NotRequired["MultiModalPlaceholderDict"]
144 """
145 Placeholder ranges for the multi-modal data.
146 """
147
145 - mm_processor_kwargs: NotRequired[Optional[Dict[str, Any]]]
148 + mm_processor_kwargs: NotRequired[Dict[str, Any]]
149 """
150 Optional multi-modal processor kwargs to be forwarded to the
151 multimodal input mapper & processor. Note that if multiple modalities
@@ -159,7 +162,7 @@ def token_inputs(
162 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
163 ) -> TokenInputs:
164 """Construct :class:`TokenInputs` from optional values."""
162 - inputs = TokenInputs(prompt_token_ids=prompt_token_ids)
165 + inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
166
167 if prompt is not None:
168 inputs["prompt"] = prompt
@@ -173,12 +176,6 @@ def token_inputs(
176 return inputs
177
178
176 -SingletonInputs = TokenInputs
177 -"""
178 -A processed :class:`SingletonPrompt` which can be passed to
179 -:class:`vllm.sequence.Sequence`.
180 -"""
181 -
179 DecoderOnlyInputs = TokenInputs
180 """
181 The inputs in :class:`~vllm.LLMEngine` before they are
@@ -187,28 +184,30 @@ This specifies the data required for decoder-only models.
184 """
185
186
190 -class EncoderDecoderInputs(TokenInputs):
187 +class EncoderDecoderInputs(TypedDict):
188 """
189 The inputs in :class:`~vllm.LLMEngine` before they are
190 passed to the model executor.
191
192 This specifies the required data for encoder-decoder models.
193 """
197 - encoder_prompt_token_ids: List[int]
198 - """The token IDs of the encoder prompt."""
194 + encoder: TokenInputs
195 + """The inputs for the encoder portion."""
196
200 - encoder_prompt: NotRequired[Optional[str]]
201 - """
202 - The original encoder prompt text corresponding to the token IDs, if
203 - available.
204 - """
197 + decoder: TokenInputs
198 + """The inputs for the decoder portion."""
199
206 - encoder_multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
207 - """
208 - Optional multi-modal data to pass to the encoder model,
209 - if the model supports it.
210 - """
200
201 +SingletonInputs = TokenInputs
202 +"""
203 +A processed :class:`SingletonPrompt` which can be passed to
204 +:class:`vllm.sequence.Sequence`.
205 +"""
206 +
207 +ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
208 +"""
209 +The inputs to :data:`vllm.inputs.InputProcessor`.
210 +"""
211
212 _T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
213 _T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
vllm/inputs/parse.py
+9 -6
@@ -4,9 +4,9 @@ from typing_extensions import TypeIs
4
5 from vllm.utils import is_list_of
6
7 -from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
8 - ExplicitEncoderDecoderPrompt, PromptType, SingletonPrompt,
9 - TextPrompt, TokensPrompt)
7 +from .data import (EncoderDecoderInputs, ExplicitEncoderDecoderPrompt,
8 + ProcessorInputs, PromptType, SingletonPrompt, TextPrompt,
9 + TokensPrompt)
10
11
12 class ParsedText(TypedDict):
@@ -98,12 +98,15 @@ def parse_singleton_prompt(
98 raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
99
100
101 +def is_token_prompt(prompt: PromptType) -> TypeIs[TokensPrompt]:
102 + return isinstance(prompt, dict) and "prompt_token_ids" in prompt
103 +
104 +
105 def is_explicit_encoder_decoder_prompt(
106 prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
107 return isinstance(prompt, dict) and "encoder_prompt" in prompt
108
109
110 def is_encoder_decoder_inputs(
107 - inputs: Union[DecoderOnlyInputs, EncoderDecoderInputs],
108 -) -> TypeIs[EncoderDecoderInputs]:
109 - return "encoder_prompt_token_ids" in inputs
111 + inputs: ProcessorInputs) -> TypeIs[EncoderDecoderInputs]:
112 + return "encoder" in inputs and "decoder" in inputs
vllm/inputs/preprocess.py
+136 -133
@@ -1,5 +1,5 @@
1 import asyncio
2 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
2 +from typing import List, Optional
3
4 from typing_extensions import assert_never
5
@@ -10,22 +10,12 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11 from vllm.utils import print_warning_once
12
13 -from .data import (DecoderOnlyInputs, EncoderDecoderInputs, PromptType,
14 - SingletonPrompt)
13 +from .data import (DecoderOnlyInputs, EncoderDecoderInputs, ProcessorInputs,
14 + PromptType, SingletonInputs, SingletonPrompt, token_inputs)
15 from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
16
17 -if TYPE_CHECKING:
18 - from vllm.multimodal import MultiModalDataDict
19 -
17 logger = init_logger(__name__)
18
22 -PromptComponents = Tuple[Optional[str], List[int],
23 - Optional["MultiModalDataDict"], Optional[Dict[str,
24 - Any]]]
25 -DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
26 - Optional["MultiModalDataDict"],
27 - Optional[Dict[str, Any]]]
28 -
19
20 class InputPreprocessor:
21
@@ -115,7 +105,7 @@ class InputPreprocessor:
105 "default" decoder prompt be <BOS>.
106
107 However, it is possible that in the future
118 - other models may have different or more
108 + other models may have different or more
109 complex logic for the default decoder prompt.
110 This motivates having a special helper method
111 for default decoder prompts.
@@ -132,7 +122,6 @@ class InputPreprocessor:
122 def _prepare_decoder_input_ids_for_generation(
123 self,
124 decoder_input_ids: Optional[List[int]],
135 - force_bos: bool = True,
125 ) -> List[int]:
126 """
127 Prepares `decoder_input_ids` for generation with encoder-decoder models.
@@ -162,8 +151,8 @@ class InputPreprocessor:
151 # use decoder_start_token_id as decoder_input_ids
152 decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
153
165 - if force_bos and (len(decoder_input_ids) == 0
166 - or decoder_input_ids[0] != decoder_start_token_id):
154 + if (len(decoder_input_ids) == 0
155 + or decoder_input_ids[0] != decoder_start_token_id):
156 decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
157
158 return decoder_input_ids
@@ -209,12 +198,12 @@ class InputPreprocessor:
198 prompt=prompt,
199 lora_request=lora_request)
200
212 - def _extract_prompt_components(
201 + def _prompt_to_llm_inputs(
202 self,
203 prompt: SingletonPrompt,
204 request_id: str,
205 lora_request: Optional[LoRARequest] = None,
217 - ) -> PromptComponents:
206 + ) -> SingletonInputs:
207 '''
208 Extract the components of any single encoder or decoder input prompt.
209
@@ -241,34 +230,52 @@ class InputPreprocessor:
230 request_id=request_id,
231 lora_request=lora_request,
232 )
244 - multi_modal_data = None
245 - mm_processor_kwargs = None
246 - elif parsed["type"] == "tokens":
247 - prompt_text = None
248 - prompt_token_ids = parsed["content"]["prompt_token_ids"]
249 - multi_modal_data = parsed["content"].get("multi_modal_data")
250 - mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
251 - elif parsed["type"] == "text":
252 - prompt_text = parsed["content"]["prompt"]
233 +
234 + return token_inputs(
235 + prompt=prompt_text,
236 + prompt_token_ids=prompt_token_ids,
237 + )
238 +
239 + if parsed["type"] == "tokens":
240 + tokens_content = parsed["content"]
241 +
242 + prompt_token_ids = tokens_content["prompt_token_ids"]
243 + multi_modal_data = tokens_content.get("multi_modal_data")
244 + mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
245 +
246 + return token_inputs(
247 + prompt_token_ids=prompt_token_ids,
248 + multi_modal_data=multi_modal_data,
249 + mm_processor_kwargs=mm_processor_kwargs,
250 + )
251 +
252 + if parsed["type"] == "text":
253 + text_content = parsed["content"]
254 +
255 + prompt_text = text_content["prompt"]
256 prompt_token_ids = self._tokenize_prompt(
257 prompt_text,
258 request_id=request_id,
259 lora_request=lora_request,
260 )
258 - multi_modal_data = parsed["content"].get("multi_modal_data")
259 - mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
260 - else:
261 - assert_never(parsed)
261 + multi_modal_data = text_content.get("multi_modal_data")
262 + mm_processor_kwargs = text_content.get("mm_processor_kwargs")
263 +
264 + return token_inputs(
265 + prompt=prompt_text,
266 + prompt_token_ids=prompt_token_ids,
267 + multi_modal_data=multi_modal_data,
268 + mm_processor_kwargs=mm_processor_kwargs,
269 + )
270
263 - return (prompt_text, prompt_token_ids, multi_modal_data,
264 - mm_processor_kwargs)
271 + assert_never(parsed)
272
266 - async def _extract_prompt_components_async(
273 + async def _prompt_to_llm_inputs_async(
274 self,
275 prompt: SingletonPrompt,
276 request_id: str,
277 lora_request: Optional[LoRARequest] = None,
271 - ) -> PromptComponents:
278 + ) -> SingletonInputs:
279 """Async version of :meth:`_extract_prompt_components`."""
280 parsed = parse_singleton_prompt(prompt)
281
@@ -279,59 +286,74 @@ class InputPreprocessor:
286 request_id=request_id,
287 lora_request=lora_request,
288 )
282 - multi_modal_data = None
283 - mm_processor_kwargs = None
284 - elif parsed["type"] == "tokens":
285 - prompt_text = None
286 - prompt_token_ids = parsed["content"]["prompt_token_ids"]
287 - multi_modal_data = parsed["content"].get("multi_modal_data")
288 - mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
289 - elif parsed["type"] == "text":
290 - prompt_text = parsed["content"]["prompt"]
289 +
290 + return token_inputs(
291 + prompt=prompt_text,
292 + prompt_token_ids=prompt_token_ids,
293 + )
294 +
295 + if parsed["type"] == "tokens":
296 + tokens_content = parsed["content"]
297 +
298 + prompt_token_ids = tokens_content["prompt_token_ids"]
299 + multi_modal_data = tokens_content.get("multi_modal_data")
300 + mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
301 +
302 + return token_inputs(
303 + prompt_token_ids=prompt_token_ids,
304 + multi_modal_data=multi_modal_data,
305 + mm_processor_kwargs=mm_processor_kwargs,
306 + )
307 +
308 + if parsed["type"] == "text":
309 + text_content = parsed["content"]
310 +
311 + prompt_text = text_content["prompt"]
312 prompt_token_ids = await self._tokenize_prompt_async(
313 prompt_text,
314 request_id=request_id,
315 lora_request=lora_request,
316 )
296 - multi_modal_data = parsed["content"].get("multi_modal_data")
297 - mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
298 - else:
299 - assert_never(parsed)
317 + multi_modal_data = text_content.get("multi_modal_data")
318 + mm_processor_kwargs = text_content.get("mm_processor_kwargs")
319 +
320 + return token_inputs(
321 + prompt=prompt_text,
322 + prompt_token_ids=prompt_token_ids,
323 + multi_modal_data=multi_modal_data,
324 + mm_processor_kwargs=mm_processor_kwargs,
325 + )
326
301 - return (prompt_text, prompt_token_ids, multi_modal_data,
302 - mm_processor_kwargs)
327 + assert_never(parsed)
328
329 def _build_enc_dec_llm_inputs(
330 self,
306 - encoder_comps: PromptComponents,
307 - decoder_comps: DecoderPromptComponents,
308 - mm_processor_kwargs: Dict[str, Any],
331 + encoder_inputs: SingletonInputs,
332 + decoder_inputs: Optional[SingletonInputs],
333 ) -> EncoderDecoderInputs:
310 - encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311 - decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312 -
313 - # Reminder: Please update docs/source/serving/compatibility_matrix.rst
314 - # If the feature combo become valid
315 - if decoder_mm_data is not None:
316 - raise ValueError(
317 - "Multi-modality decoder inputs of encoder-decoder models are "
318 - "not supported yet")
319 -
320 - # For Multi-Modal models (e.g., mllama), the text input can be
321 - # <|image|><|begin_of_text|>hello world. And we should not add
322 - # another <|begin_of_text|> to the beginning.
323 - decoder_prompt_ids = (self._prepare_decoder_input_ids_for_generation(
324 - decoder_prompt_ids,
325 - force_bos=(encoder_mm_data is None and decoder_mm_data is None)))
334 + if encoder_inputs["type"] == "token":
335 + pass
336 + else:
337 + assert_never(encoder_inputs)
338 +
339 + if decoder_inputs is None:
340 + dec_token_ids = self._prepare_decoder_input_ids_for_generation(
341 + None)
342 + decoder_inputs = token_inputs(dec_token_ids)
343 + elif decoder_inputs["type"] == "token":
344 + dec_token_ids = self._prepare_decoder_input_ids_for_generation(
345 + decoder_inputs["prompt_token_ids"])
346 + decoder_inputs["prompt_token_ids"] = dec_token_ids
347 +
348 + if "multi_modal_data" in decoder_inputs:
349 + raise ValueError("Multi-modal decoder inputs of encoder-"
350 + "decoder models are not supported yet")
351 + else:
352 + assert_never(encoder_inputs)
353
354 return EncoderDecoderInputs(
328 - prompt_token_ids=decoder_prompt_ids,
329 - prompt=decoder_prompt,
330 - multi_modal_data=decoder_mm_data,
331 - mm_processor_kwargs=mm_processor_kwargs,
332 - encoder_prompt_token_ids=encoder_prompt_ids,
333 - encoder_prompt=encoder_prompt,
334 - encoder_multi_modal_data=encoder_mm_data,
355 + encoder=encoder_inputs,
356 + decoder=decoder_inputs,
357 )
358
359 def _process_encoder_decoder_prompt(
@@ -341,8 +363,7 @@ class InputPreprocessor:
363 ) -> EncoderDecoderInputs:
364 '''
365 For encoder/decoder models only:
344 - Process an input prompt into an
345 - :class:`EncoderDecoderInputs` instance.
366 + Process an input prompt into an :class:`EncoderDecoderInputs` instance.
367
368 There are two types of input prompts:
369 singleton prompts which carry only the
@@ -361,7 +382,7 @@ class InputPreprocessor:
382 have any possible singleton type; thus this
383 method relies on helper functions to obtain
384 token ids for the sub-prompts.
364 -
385 +
386 Arguments:
387
388 * prompt: an input prompt
@@ -372,40 +393,31 @@ class InputPreprocessor:
393 * :class:`EncoderDecoderInputs` instance
394 '''
395
375 - encoder_comps: PromptComponents
376 - decoder_comps: DecoderPromptComponents
396 + encoder_inputs: SingletonInputs
397 + decoder_inputs: Optional[SingletonInputs]
398
399 if is_explicit_encoder_decoder_prompt(prompt):
379 - encoder_comps = self._extract_prompt_components(
400 + encoder_inputs = self._prompt_to_llm_inputs(
401 prompt["encoder_prompt"],
402 request_id=request_id,
403 )
404
405 if (decoder_input := prompt["decoder_prompt"]) is None:
385 - decoder_comps = None, None, None, None
406 + decoder_inputs = None
407 else:
387 - decoder_comps = self._extract_prompt_components(
408 + decoder_inputs = self._prompt_to_llm_inputs(
409 decoder_input,
410 request_id=request_id,
411 )
391 - # Handle this carefully in case it was directly initialized by user
392 - mm_processor_kwargs = prompt.get("mm_processor_kwargs", {})
412 else:
394 - encoder_comps = self._extract_prompt_components(
413 + encoder_inputs = self._prompt_to_llm_inputs(
414 prompt,
415 request_id=request_id,
416 )
398 - # If there are no decoder components, we assume the
399 - # mm_processor_kwargs are in the encoder prompt
400 - mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
401 - -1] is not None else {}
402 - decoder_comps = None, None, None, None
403 -
404 - return self._build_enc_dec_llm_inputs(
405 - encoder_comps,
406 - decoder_comps,
407 - mm_processor_kwargs,
408 - )
417 +
418 + decoder_inputs = None
419 +
420 + return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
421
422 async def _process_encoder_decoder_prompt_async(
423 self,
@@ -413,59 +425,50 @@ class InputPreprocessor:
425 request_id: str,
426 ) -> EncoderDecoderInputs:
427 """Async version of :meth:`_process_encoder_decoder_prompt`."""
416 - encoder_comps: PromptComponents
417 - decoder_comps: DecoderPromptComponents
428 + encoder_inputs: SingletonInputs
429 + decoder_inputs: Optional[SingletonInputs]
430
431 if is_explicit_encoder_decoder_prompt(prompt):
420 - encoder_task = self._extract_prompt_components_async(
432 + encoder_task = self._prompt_to_llm_inputs_async(
433 prompt["encoder_prompt"],
434 request_id=request_id,
435 )
436
437 if (decoder_input := prompt["decoder_prompt"]) is None:
426 - encoder_comps = await encoder_task
427 - decoder_comps = None, None, None, None
438 + encoder_inputs = await encoder_task
439 + decoder_inputs = None
440 else:
429 - decoder_task = self._extract_prompt_components_async(
441 + decoder_task = self._prompt_to_llm_inputs_async(
442 decoder_input,
443 request_id=request_id,
444 )
445
434 - encoder_comps, decoder_comps = await asyncio.gather(
446 + encoder_inputs, decoder_inputs = await asyncio.gather(
447 encoder_task, decoder_task)
436 - mm_processor_kwargs = prompt["mm_processor_kwargs"]
448 else:
438 - encoder_comps = await self._extract_prompt_components_async(
449 + encoder_inputs = await self._prompt_to_llm_inputs_async(
450 prompt,
451 request_id=request_id,
452 )
442 - # If there are no decoder components, we assume the
443 - # mm_processor_kwargs are in the encoder prompt
444 - mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
445 - -1] is not None else {}
446 - decoder_comps = None, None, None, None
447 -
448 - return self._build_enc_dec_llm_inputs(
449 - encoder_comps,
450 - decoder_comps,
451 - mm_processor_kwargs,
452 - )
453 +
454 + decoder_inputs = None
455 +
456 + return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
457
458 def _build_decoder_only_llm_inputs(
459 self,
456 - prompt_comps: PromptComponents,
460 + prompt_inputs: DecoderOnlyInputs,
461 prompt_adapter_request: Optional[PromptAdapterRequest],
462 ) -> DecoderOnlyInputs:
459 - (prompt, prompt_token_ids, multi_modal_data,
460 - mm_processor_kwargs) = prompt_comps
461 -
462 - prompt_token_ids = self._apply_prompt_adapter(
463 - prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
463 + if prompt_inputs["type"] == "token":
464 + prompt_inputs["prompt_token_ids"] = self._apply_prompt_adapter(
465 + prompt_inputs["prompt_token_ids"],
466 + prompt_adapter_request=prompt_adapter_request,
467 + )
468 + else:
469 + assert_never(prompt_inputs)
470
465 - return DecoderOnlyInputs(prompt_token_ids=prompt_token_ids,
466 - prompt=prompt,
467 - multi_modal_data=multi_modal_data,
468 - mm_processor_kwargs=mm_processor_kwargs)
471 + return prompt_inputs
472
473 def _process_decoder_only_prompt(
474 self,
@@ -490,7 +493,7 @@ class InputPreprocessor:
493 * :class:`DecoderOnlyInputs` instance
494 '''
495
493 - prompt_comps = self._extract_prompt_components(
496 + prompt_comps = self._prompt_to_llm_inputs(
497 prompt,
498 request_id=request_id,
499 lora_request=lora_request,
@@ -509,7 +512,7 @@ class InputPreprocessor:
512 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
513 ) -> DecoderOnlyInputs:
514 """Async version of :meth:`_process_decoder_only_prompt`."""
512 - prompt_comps = await self._extract_prompt_components_async(
515 + prompt_comps = await self._prompt_to_llm_inputs_async(
516 prompt,
517 request_id=request_id,
518 lora_request=lora_request,
@@ -526,7 +529,7 @@ class InputPreprocessor:
529 request_id: str,
530 lora_request: Optional[LoRARequest] = None,
531 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
529 - ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
532 + ) -> ProcessorInputs:
533 """Preprocess the input prompt."""
534 if self.is_encoder_decoder_model():
535 # Encoder-decoder model requires special mapping of
@@ -554,7 +557,7 @@ class InputPreprocessor:
557 request_id: str,
558 lora_request: Optional[LoRARequest] = None,
559 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
557 - ) -> Union[DecoderOnlyInputs, EncoderDecoderInputs]:
560 + ) -> ProcessorInputs:
561 """Async version of :meth:`preprocess`."""
562 if self.is_encoder_decoder_model():
563 # Encoder-decoder model requires special mapping of
vllm/inputs/registry.py
+7 -7
@@ -2,7 +2,7 @@ import functools
2 from collections import UserDict
3 from dataclasses import dataclass
4 from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5 - Optional, Protocol, Type)
5 + Optional, Protocol, Type, cast)
6
7 from torch import nn
8 from transformers import PretrainedConfig
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
12 from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
13 resolve_mm_processor_kwargs)
14
15 -from .data import DecoderOnlyInputs
15 +from .data import ProcessorInputs
16
17 if TYPE_CHECKING:
18 from vllm.config import ModelConfig
@@ -109,7 +109,7 @@ class _MultiModalCounts(UserDict):
109 raise KeyError(msg) from exc
110
111
112 -InputProcessor = Callable[[InputContext, DecoderOnlyInputs], DecoderOnlyInputs]
112 +InputProcessor = Callable[[InputContext, ProcessorInputs], ProcessorInputs]
113 """Preprocess the inputs to the model."""
114
115
@@ -254,8 +254,8 @@ class InputRegistry:
254 def _default_input_processor(
255 self,
256 ctx: InputContext,
257 - inputs: DecoderOnlyInputs,
258 - ) -> DecoderOnlyInputs:
257 + inputs: ProcessorInputs,
258 + ) -> ProcessorInputs:
259 """The default input processor is a no-op."""
260 return inputs
261
@@ -288,7 +288,7 @@ class InputRegistry:
288 .get(model_cls, self._default_input_processor)
289
290 def process_input(self, model_config: "ModelConfig",
291 - inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
291 + inputs: ProcessorInputs) -> ProcessorInputs:
292 """
293 Apply an input processor to an instance of model inputs.
294
@@ -308,7 +308,7 @@ class InputRegistry:
308 # If it's empty, it'll fall back to the default kwarg values
309 mm_processor_kwargs = resolve_mm_processor_kwargs(
310 model_config.mm_processor_kwargs,
311 - inputs.get("mm_processor_kwargs"),
311 + cast(Dict[str, Any], inputs.get("mm_processor_kwargs")),
312 processor,
313 )
314
vllm/model_executor/models/mllama.py
+67 -29
@@ -36,8 +36,8 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
36 from vllm.attention.ops.paged_attn import PagedAttention
37 from vllm.config import CacheConfig, MultiModalConfig
38 from vllm.distributed import get_tensor_model_parallel_world_size
39 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
40 - EncoderDecoderInputs, InputContext)
39 +from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
40 + InputContext, TokenInputs, token_inputs)
41 from vllm.logger import init_logger
42 from vllm.model_executor.layers.layernorm import RMSNorm
43 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -52,6 +52,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52 from vllm.model_executor.sampling_metadata import SamplingMetadata
53 from vllm.multimodal import MULTIMODAL_REGISTRY
54 from vllm.sequence import SequenceData
55 +from vllm.utils import is_list_of
56
57 from .clip import CLIPMLP
58 from .interfaces import SupportsMultiModal
@@ -86,41 +87,58 @@ def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
87 return num_images
88
89
89 -def input_processor_for_mllama(ctx: InputContext,
90 - inputs: Union[DecoderOnlyInputs,
91 - EncoderDecoderInputs]):
92 - # move encoder_prompt to prompt
93 - if inputs.get("prompt") is None:
94 - inputs["prompt"] = inputs["encoder_prompt"]
95 - inputs["prompt_token_ids"] = inputs["encoder_prompt_token_ids"]
90 +def input_processor_for_mllama(
91 + ctx: InputContext,
92 + inputs: EncoderDecoderInputs,
93 +) -> EncoderDecoderInputs:
94 + # Example input to processor:
95 + # {
96 + # 'encoder': {
97 + # 'type': 'token',
98 + # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
99 + # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
100 + # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
101 + # },
102 + # 'decoder': {
103 + # 'type': 'token',
104 + # 'prompt_token_ids': [128000],
105 + # },
106 + # }
107 +
108 + # move encoder prompt to decoder
109 + dec_inputs = TokenInputs(**inputs["encoder"])
110 +
111 + multi_modal_data = dec_inputs.get("multi_modal_data")
112 + if multi_modal_data is None or "image" not in multi_modal_data:
113 + # text-only
114 + return EncoderDecoderInputs(
115 + encoder=token_inputs([]),
116 + decoder=dec_inputs,
117 + )
118
97 - # process multi-modal data
98 - multi_modal_data = inputs.get("encoder_multi_modal_data")
119 + image_data = multi_modal_data["image"]
120 + if isinstance(image_data, Image.Image):
121 + image_data = [image_data]
122
100 - if multi_modal_data is None or "image" not in multi_modal_data \
101 - or multi_modal_data["image"] is None:
102 - # text-only
103 - inputs["encoder_prompt"] = ""
104 - inputs["encoder_prompt_token_ids"] = []
105 - inputs["encoder_multi_modal_data"] = {}
106 - return inputs
123 + assert is_list_of(image_data, Image.Image)
124
108 - if isinstance(multi_modal_data['image'], Image.Image):
109 - multi_modal_data['image'] = [multi_modal_data['image']]
125 # Since only the last group of consecutive images
126 # are attended by the decoded tokens, we only need to
127 # get the number of tiles for those images.
128 num_decode_images = _get_num_image_in_last_group(
114 - inputs["prompt_token_ids"])
129 + dec_inputs["prompt_token_ids"])
130 +
131 hf_config = ctx.model_config.hf_config
132 + vision_config = hf_config.vision_config
133 +
134 num_tiles = 0
117 - for image in multi_modal_data["image"][::-1]:
135 + for image in image_data[::-1]:
136 width, height = image.size
119 - tile_size = hf_config.vision_config.image_size
137 + tile_size = vision_config.image_size
138 canvas_height, canvas_width = get_optimal_tiled_canvas(
139 image_height=height,
140 image_width=width,
123 - max_image_tiles=hf_config.vision_config.max_num_tiles,
141 + max_image_tiles=vision_config.max_num_tiles,
142 tile_size=tile_size,
143 )
144 num_tiles_height = canvas_height // tile_size
@@ -133,14 +151,34 @@ def input_processor_for_mllama(ctx: InputContext,
151 # Set encoder prompt length based on the number of tiles.
152 # This tells the block manager to allocate correct number
153 # of slots for encoder tokens.
136 - assert hf_config.vision_config.image_size % 14 == 0, \
154 + assert vision_config.image_size % 14 == 0, \
155 "chunk size should be multiple of 14"
138 - token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
156 + token_per_chunk = (vision_config.image_size // 14)**2 + 1
157 num_tokens = num_tiles * token_per_chunk
140 - inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
141 - inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID] * num_tokens
158
143 - return inputs
159 + # Example output from processor:
160 + # {
161 + # 'encoder': {
162 + # 'type': 'token',
163 + # 'prompt_token_ids': [128256, 128256, ..., 128256],
164 + # 'prompt': '<|image|><|image|>...<|image|>',
165 + # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
166 + # },
167 + # 'decoder': {
168 + # 'type': 'token',
169 + # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
170 + # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
171 + # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
172 + # },
173 + # }
174 + return EncoderDecoderInputs(
175 + encoder=token_inputs(
176 + prompt_token_ids=[MLLAMA_IMAGE_TOKEN_ID] * num_tokens,
177 + prompt=MLLAMA_IMAGE_TOKEN * num_tokens,
178 + multi_modal_data=multi_modal_data,
179 + ),
180 + decoder=dec_inputs,
181 + )
182
183
184 def get_max_mllama_image_tokens(ctx: InputContext) -> int:
vllm/model_executor/models/registry.py
+5
@@ -343,6 +343,11 @@ class _ModelRegistry:
343 def _raise_for_unsupported(self, architectures: List[str]):
344 all_supported_archs = self.get_supported_archs()
345
346 + if any(arch in all_supported_archs for arch in architectures):
347 + raise ValueError(
348 + f"Model architectures {architectures} failed "
349 + "to be inspected. Please check the logs for more details.")
350 +
351 raise ValueError(
352 f"Model architectures {architectures} are not supported for now. "
353 f"Supported architectures: {all_supported_archs}")
vllm/sequence.py
+44 -69
@@ -9,12 +9,12 @@ from functools import cached_property, reduce
9 from typing import (TYPE_CHECKING, Any, Callable, DefaultDict, Dict, List,
10 Mapping, Optional)
11 from typing import Sequence as GenericSequence
12 -from typing import Set, Tuple, Union, cast
12 +from typing import Set, Tuple, Union
13
14 import msgspec
15 import torch
16 +from typing_extensions import assert_never
17
17 -from vllm.inputs.parse import is_encoder_decoder_inputs
18 from vllm.lora.request import LoRARequest
19 from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
20 from vllm.pooling_params import PoolingParams
@@ -379,15 +379,10 @@ class SequenceData(msgspec.Struct,
379
380 class Sequence:
381 """Stores the data, status, and block information of a sequence.
382 -
383 - The sequence is constructed from the :code:`SingletonInputs` instance
384 - passed in through the :code:`inputs` constructor argument.
385 -
386 - For encoder/decoder models, SingletonInputs encapsulates both a
387 - decoder and encoder prompt, creating an ambiguity about which
388 - prompt to construct the sequence from. The `from_decoder_prompt`
389 - constructor argument signals whether to construct the Sequence
390 - from the SingletonInputs decoder prompt, or encoder prompt.
382 +
383 + The sequence is constructed from the :data:`DecoderOnlyInputs`
384 + (for decoder-only) or :data:`EncoderDecoderInputs` (for encoder-decoder)
385 + instance passed in through the :code:`inputs` constructor argument.
386
387 Args:
388 seq_id: The ID of the sequence.
@@ -397,10 +392,6 @@ class Sequence:
392 eos_token_id: The end-of-sequence (EOS) token id recognized by this LLM.
393 lora_request: LoRA request.
394 prompt_adapter_request: Prompt Adapter request.
400 - from_decoder_prompt: Construct Sequence from SingletonInputs decoder
401 - prompt (True) or encoder prompt (False.) Must be
402 - True for decoder-only model.
403 -
395 """
396
397 def __init__(
@@ -411,7 +402,6 @@ class Sequence:
402 eos_token_id: Optional[int] = None,
403 lora_request: Optional[LoRARequest] = None,
404 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
414 - from_decoder_prompt: bool = True,
405 ) -> None:
406 self.seq_id = seq_id
407 self.inputs = inputs
@@ -419,33 +409,6 @@ class Sequence:
409 self.eos_token_id = eos_token_id
410 self.lora_request = lora_request
411 self.prompt_adapter_request = prompt_adapter_request
422 - self.from_decoder_prompt = from_decoder_prompt
423 -
424 - # For decoder-only models, a Sequence is constructed
425 - # from an DecoderOnlyInputs instance (the `inputs` arg.)
426 - #
427 - # For encoder/decoder models the same `inputs`
428 - # instance could be utilized to construct either an
429 - # encoder sequence or a decoder sequence, because
430 - # `DecoderOnlyInputs` has both decoder- and encoder-oriented
431 - # member variables (i.e. it encapsulates both an encoder
432 - # and a decoder prompt.) The decision of which type of sequence
433 - # to generate is determined by the `from_decoder_prompt` argument.
434 - #
435 - # When constructing a encoder sequence
436 - # (`from_decoder_prompt` False) it matters that
437 - # the `DecoderOnlyInputs` instance stored in `inputs` is valid
438 - # in the sense that its encoder-related member variables are
439 - # populated; below, an exception is raised if this is
440 - # not the case.
441 - #
442 - # When constructing a decoder sequence (`from_decoder_prompt` True)
443 - # it does not matter whether `inputs` has its encoder-related
444 - # member variables populated.
445 - if not (from_decoder_prompt or is_encoder_decoder_inputs(inputs)):
446 - raise ValueError("Cannot extract encoder input prompt from "
447 - f"invalid input {inputs}; did you forget the "
448 - "encoder input prompt fields?")
412
413 self.data = SequenceData.from_seqs(self.prompt_token_ids)
414 self.output_logprobs: SampleLogprobs = []
@@ -470,45 +433,57 @@ class Sequence:
433
434 @cached_property
435 def prompt(self) -> Optional[str]:
473 - # Select decoder or encoder input prompt str, as appropriate
474 - prompt_key: str = ("prompt"
475 - if self.from_decoder_prompt else "encoder_prompt")
436 + inputs = self.inputs
437
477 - return cast(Optional[str], self.inputs.get(prompt_key))
438 + if inputs["type"] == "token":
439 + return inputs.get("prompt")
440 +
441 + assert_never(inputs)
442
443 @cached_property
444 def prompt_token_ids(self) -> List[int]:
481 - # Select decoder or encoder input prompt token ids, as appropriate
482 - prompt_token_ids_key: str = ("prompt_token_ids"
483 - if self.from_decoder_prompt else
484 - "encoder_prompt_token_ids")
445 + inputs = self.inputs
446
486 - # Cache computed prompt token ids
487 - return cast(List[int], self.inputs.get(prompt_token_ids_key))
447 + if inputs["type"] == "token":
448 + return inputs.get("prompt_token_ids", [])
449
489 - @property
490 - def multi_modal_data(self) -> MultiModalDataDict:
450 + assert_never(inputs)
451 +
452 + @cached_property
453 + def prompt_embeds(self) -> Optional[torch.Tensor]:
454 inputs = self.inputs
455
493 - if (inputs.get("multi_modal_data")
494 - and inputs.get("encoder_multi_modal_data")):
495 - raise ValueError(
496 - "Multi-modal data in both encoder and decoder is not supported."
497 - )
456 + if inputs["type"] == "token":
457 + return None
458
499 - return cast(
500 - MultiModalDataDict,
501 - (inputs.get("multi_modal_data")
502 - or inputs.get("encoder_multi_modal_data") or {}),
503 - )
459 + assert_never(inputs)
460 +
461 + @cached_property
462 + def multi_modal_data(self) -> "MultiModalDataDict":
463 + inputs = self.inputs
464 +
465 + if inputs["type"] == "token":
466 + return inputs.get("multi_modal_data", {})
467 +
468 + assert_never(inputs)
469 +
470 + @cached_property
471 + def mm_processor_kwargs(self) -> Dict[str, Any]:
472 + inputs = self.inputs
473 +
474 + if inputs["type"] == "token":
475 + return inputs.get("mm_processor_kwargs", {})
476 +
477 + assert_never(inputs)
478
479 @property
480 def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
507 - return self.inputs.get("multi_modal_placeholders") or {}
481 + inputs = self.inputs
482
509 - @property
510 - def mm_processor_kwargs(self) -> Dict[str, Any]:
511 - return self.inputs.get("mm_processor_kwargs") or {}
483 + if inputs["type"] == "token":
484 + return inputs.get("multi_modal_placeholders", {})
485 +
486 + assert_never(inputs)
487
488 @property
489 def lora_int_id(self) -> int: