@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7eb4a51c5f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Support serving encoder/decoder models (#7258)

Cyrus Leung committed Aug 9, 2024 at 10:39 UTC 7eb4a51c5f34a52427d170d0e654e0a346c6d69c
25 files changed +603 -464
.github/workflows/mypy.yaml
+1 -1
@@ -25,7 +25,7 @@ jobs:
25 - name: Install dependencies
26 run: |
27 python -m pip install --upgrade pip
28 - pip install mypy==1.9.0
28 + pip install mypy==1.11.1
29 pip install types-setuptools
30 pip install types-PyYAML
31 pip install types-requests
examples/offline_inference_encoder_decoder.py
+4 -4
@@ -4,8 +4,8 @@ encoder/decoder models, specifically BART
4 '''
5
6 from vllm import LLM, SamplingParams
7 -from vllm.inputs import ExplicitEncoderDecoderPrompt, TextPrompt, TokensPrompt
8 -from vllm.utils import zip_enc_dec_prompt_lists
7 +from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
8 + TokensPrompt, zip_enc_dec_prompts)
9
10 dtype = "float"
11
@@ -61,9 +61,9 @@ enc_dec_prompt3 = ExplicitEncoderDecoderPrompt(
61 )
62
63 # - Finally, here's a useful helper function for zipping encoder and
64 -# decoder prompt lists together into a list of ExplicitEncoderDecoderPrompt
64 +# decoder prompts together into a list of ExplicitEncoderDecoderPrompt
65 # instances
66 -zipped_prompt_list = zip_enc_dec_prompt_lists(
66 +zipped_prompt_list = zip_enc_dec_prompts(
67 ['An encoder prompt', 'Another encoder prompt'],
68 ['A decoder prompt', 'Another decoder prompt'])
69
requirements-common.txt
+1 -1
@@ -19,7 +19,7 @@ prometheus-fastapi-instrumentator >= 7.0.0
19 tiktoken >= 0.6.0 # Required for DBRX tokenizer
20 lm-format-enforcer == 0.10.3
21 outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
22 -typing_extensions
22 +typing_extensions >= 4.10
23 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
24 pyzmq
25 gguf == 0.9.1
requirements-lint.txt
+1 -1
@@ -8,7 +8,7 @@ isort==5.13.2
8 clang-format==18.1.5
9
10 # type checking
11 -mypy==1.9.0
11 +mypy==1.11.1
12 types-PyYAML
13 types-requests
14 types-setuptools
tests/conftest.py
+19 -13
@@ -3,6 +3,7 @@ import gc
3 import os
4 import sys
5 from collections import UserList
6 +from enum import Enum
7 from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union
8
9 import pytest
@@ -14,20 +15,19 @@ from transformers import (AutoModelForCausalLM, AutoModelForSeq2SeqLM,
15 AutoModelForVision2Seq, AutoTokenizer, BatchEncoding,
16 BatchFeature)
17
17 -from tests.models.utils import DecoderPromptType
18 from vllm import LLM, SamplingParams
19 from vllm.assets.image import ImageAsset
20 from vllm.config import TokenizerPoolConfig
21 from vllm.connections import global_http_connection
22 from vllm.distributed import (destroy_distributed_environment,
23 destroy_model_parallel)
24 -from vllm.inputs import TextPrompt
24 +from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
25 + to_enc_dec_tuple_list, zip_enc_dec_prompts)
26 from vllm.logger import init_logger
27 from vllm.outputs import RequestOutput
28 from vllm.sequence import SampleLogprobs
29 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
29 - is_cpu, to_enc_dec_tuple_list,
30 - zip_enc_dec_prompt_lists)
30 + is_cpu)
31
32 logger = init_logger(__name__)
33
@@ -124,10 +124,16 @@ def example_prompts() -> List[str]:
124 return prompts
125
126
127 +class DecoderPromptType(Enum):
128 + """For encoder/decoder models only."""
129 + CUSTOM = 1
130 + NONE = 2
131 + EMPTY_STR = 3
132 +
133 +
134 @pytest.fixture
128 -def example_encoder_decoder_prompts() \
129 - -> Dict[DecoderPromptType,
130 - Tuple[List[str], List[Optional[str]]]]:
135 +def example_encoder_decoder_prompts(
136 +) -> Dict[DecoderPromptType, List[ExplicitEncoderDecoderPrompt]]:
137 '''
138 Returns an encoder prompt list and a decoder prompt list, wherein each pair
139 of same-index entries in both lists corresponds to an (encoder prompt,
@@ -150,11 +156,11 @@ def example_encoder_decoder_prompts() \
156 # NONE decoder prompt type
157 return {
158 DecoderPromptType.NONE:
153 - zip_enc_dec_prompt_lists(encoder_prompts, none_decoder_prompts),
159 + zip_enc_dec_prompts(encoder_prompts, none_decoder_prompts),
160 DecoderPromptType.EMPTY_STR:
155 - zip_enc_dec_prompt_lists(encoder_prompts, empty_str_decoder_prompts),
161 + zip_enc_dec_prompts(encoder_prompts, empty_str_decoder_prompts),
162 DecoderPromptType.CUSTOM:
157 - zip_enc_dec_prompt_lists(encoder_prompts, custom_decoder_prompts),
163 + zip_enc_dec_prompts(encoder_prompts, custom_decoder_prompts),
164 }
165
166
@@ -444,7 +450,7 @@ class HfRunner:
450
451 def generate_encoder_decoder_greedy_logprobs_limit(
452 self,
447 - encoder_decoder_prompts: Tuple[List[str], List[str]],
453 + encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
454 max_tokens: int,
455 num_logprobs: int,
456 **kwargs: Any,
@@ -608,7 +614,7 @@ class VllmRunner:
614
615 def generate_encoder_decoder_w_logprobs(
616 self,
611 - encoder_decoder_prompts: Tuple[List[str], List[str]],
617 + encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
618 sampling_params: SamplingParams,
619 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
620 '''
@@ -653,7 +659,7 @@ class VllmRunner:
659
660 def generate_encoder_decoder_greedy_logprobs(
661 self,
656 - encoder_decoder_prompts: Tuple[List[str], List[str]],
662 + encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
663 max_tokens: int,
664 num_logprobs: int,
665 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
tests/distributed/test_basic_distributed_correctness_enc_dec.py
+1 -1
@@ -11,9 +11,9 @@ pytest distributed/test_basic_distributed_correctness_enc_dec.py
11
12 import pytest
13
14 -from tests.models.utils import DecoderPromptType
14 from vllm.utils import cuda_device_count_stateless
15
16 +from ..conftest import DecoderPromptType
17 from ..models.utils import check_logprobs_close
18 from ..utils import fork_new_process_for_each_test
19
tests/entrypoints/openai/test_encoder_decoder.py new
+50
@@ -0,0 +1,50 @@
1 +import openai
2 +import pytest
3 +
4 +from ...utils import RemoteOpenAIServer
5 +
6 +MODEL_NAME = "facebook/bart-base"
7 +
8 +
9 +@pytest.fixture(scope="module")
10 +def server():
11 + args = [
12 + "--dtype",
13 + "bfloat16",
14 + "--enforce-eager",
15 + ]
16 +
17 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
18 + yield remote_server
19 +
20 +
21 +@pytest.fixture(scope="module")
22 +def client(server):
23 + return server.get_async_client()
24 +
25 +
26 +@pytest.mark.asyncio
27 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
28 +async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
29 + completion = await client.completions.create(model=model_name,
30 + prompt="Hello, my name is",
31 + max_tokens=5,
32 + temperature=0.0)
33 +
34 + assert completion.id is not None
35 + assert completion.choices is not None and len(completion.choices) == 1
36 +
37 + choice = completion.choices[0]
38 + assert len(choice.text) >= 5
39 + assert choice.finish_reason == "length"
40 + assert completion.usage == openai.types.CompletionUsage(
41 + completion_tokens=5, prompt_tokens=2, total_tokens=7)
42 +
43 + # test using token IDs
44 + completion = await client.completions.create(
45 + model=model_name,
46 + prompt=[0, 0, 0, 0, 0],
47 + max_tokens=5,
48 + temperature=0.0,
49 + )
50 + assert len(completion.choices[0].text) >= 1
tests/models/test_bart.py
+27 -11
@@ -2,6 +2,8 @@
2
3 Run `pytest tests/models/test_bart.py`.
4 """
5 +from typing import List, Optional, Tuple
6 +
7 from vllm.utils import is_cpu
8
9 if not is_cpu():
@@ -11,22 +13,31 @@ if not is_cpu():
13
14 import pytest
15
14 - from tests.models.utils import DecoderPromptType
16 + from vllm.sequence import SampleLogprobs
17
18 + from ..conftest import DecoderPromptType
19 from .utils import check_logprobs_close
20
21 MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
22
20 - DECODER_PROMPT_TYPES = ([
21 - DecoderPromptType.CUSTOM, DecoderPromptType.EMPTY_STR,
22 - DecoderPromptType.NONE
23 - ])
23 + def vllm_to_hf_output(
24 + vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
25 + decoder_prompt_type: DecoderPromptType,
26 + ):
27 + """Sanitize vllm output to be comparable with hf output."""
28 + output_ids, output_str, out_logprobs = vllm_output
29 +
30 + hf_output_str = output_str + "</s>"
31 + if decoder_prompt_type == DecoderPromptType.NONE:
32 + hf_output_str = "<s>" + hf_output_str
33 +
34 + return output_ids, hf_output_str, out_logprobs
35
36 @pytest.mark.parametrize("model", MODELS)
37 @pytest.mark.parametrize("dtype", ["float", "bfloat16"])
38 @pytest.mark.parametrize("max_tokens", [64])
39 @pytest.mark.parametrize("num_logprobs", [5])
29 - @pytest.mark.parametrize("decoder_prompt_type", DECODER_PROMPT_TYPES)
40 + @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
41 def test_models(
42 hf_runner,
43 vllm_runner,
@@ -146,8 +157,13 @@ if not is_cpu():
157 hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
158 else 0)
159
149 - check_logprobs_close(outputs_0_lst=hf_outputs,
150 - outputs_1_lst=vllm_outputs,
151 - name_0="hf",
152 - name_1="vllm",
153 - num_outputs_0_skip_tokens=hf_skip_tokens)
160 + check_logprobs_close(
161 + outputs_0_lst=hf_outputs,
162 + outputs_1_lst=[
163 + vllm_to_hf_output(vllm_output, decoder_prompt_type)
164 + for vllm_output in vllm_outputs
165 + ],
166 + name_0="hf",
167 + name_1="vllm",
168 + num_outputs_0_skip_tokens=hf_skip_tokens,
169 + )
tests/models/utils.py
-11
@@ -1,5 +1,4 @@
1 import warnings
2 -from enum import Enum
2 from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4 from vllm.sequence import SampleLogprobs
@@ -136,13 +135,3 @@ def check_logprobs_close(
135 warnings.simplefilter("always")
136
137 warnings.warn(fail_msg, stacklevel=2)
139 -
140 -
141 -class DecoderPromptType(Enum):
142 - '''
143 - For encoder/decoder models only -
144 -
145 - '''
146 - CUSTOM = 1
147 - NONE = 2
148 - EMPTY_STR = 3
tests/test_inputs.py
+1 -1
@@ -2,7 +2,7 @@ from typing import List
2
3 import pytest
4
5 -from vllm.inputs import parse_and_batch_prompt
5 +from vllm.inputs.parse import parse_and_batch_prompt
6
7 STRING_INPUTS = [
8 '',
vllm/config.py
+10
@@ -464,6 +464,16 @@ class ModelConfig:
464 if t != "attention"
465 ])
466
467 + @property
468 + def is_encoder_decoder_model(self) -> bool:
469 + """Extract the HF encoder/decoder model flag."""
470 + return getattr(self.hf_config, "is_encoder_decoder", False)
471 +
472 + @property
473 + def is_embedding_model(self) -> bool:
474 + """Extract the embedding model flag."""
475 + return self.embedding_mode
476 +
477
478 class CacheConfig:
479 """Configuration for the KV cache.
vllm/engine/async_llm_engine.py
+130 -24
@@ -5,6 +5,7 @@ from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
5 Optional, Set, Tuple, Type, Union)
6
7 from transformers import PreTrainedTokenizer
8 +from typing_extensions import assert_never
9
10 import vllm.envs as envs
11 from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
@@ -12,11 +13,14 @@ from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
13 from vllm.core.scheduler import SchedulerOutputs
14 from vllm.engine.arg_utils import AsyncEngineArgs
15 from vllm.engine.async_timeout import asyncio_timeout
15 -from vllm.engine.llm_engine import LLMEngine
16 +from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
17 + PromptComponents)
18 from vllm.engine.metrics import StatLoggerBase
19 from vllm.executor.executor_base import ExecutorAsyncBase
20 from vllm.executor.ray_utils import initialize_ray_cluster, ray
19 -from vllm.inputs import LLMInputs, PromptInputs
21 +from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
22 + SingletonPromptInputs)
23 +from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
24 from vllm.logger import init_logger
25 from vllm.lora.request import LoRARequest
26 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -293,38 +297,138 @@ class _AsyncLLMEngine(LLMEngine):
297 """Stop the remote worker execution loop."""
298 await self.model_executor.stop_remote_worker_execution_loop_async()
299
296 - async def process_model_inputs_async(
300 + async def _tokenize_prompt_async(
301 + self,
302 + prompt: str,
303 + request_id: str,
304 + lora_request: Optional[LoRARequest],
305 + ) -> List[int]:
306 + """Async version of :meth:`_tokenize_prompt`."""
307 + tokenizer = self.get_tokenizer_group("prompts must be None if "
308 + "skip_tokenizer_init is True")
309 +
310 + return await tokenizer.encode_async(request_id=request_id,
311 + prompt=prompt,
312 + lora_request=lora_request)
313 +
314 + async def _extract_prompt_components_async(
315 self,
316 + inputs: SingletonPromptInputs,
317 request_id: str,
318 + lora_request: Optional[LoRARequest] = None,
319 + ) -> PromptComponents:
320 + """Async version of :meth:`_extract_prompt_components`."""
321 + if isinstance(inputs, str):
322 + prompt = inputs
323 + prompt_token_ids = await self._tokenize_prompt_async(
324 + prompt,
325 + request_id=request_id,
326 + lora_request=lora_request,
327 + )
328 + multi_modal_data = None
329 + elif isinstance(inputs, dict):
330 + if "prompt_token_ids" in inputs:
331 + prompt = None
332 + prompt_token_ids = inputs["prompt_token_ids"]
333 + else:
334 + # NOTE: This extra assignment is required to pass mypy
335 + prompt = parsed_prompt = inputs["prompt"]
336 + prompt_token_ids = await self._tokenize_prompt_async(
337 + parsed_prompt,
338 + request_id=request_id,
339 + lora_request=lora_request,
340 + )
341 +
342 + multi_modal_data = inputs.get("multi_modal_data")
343 + else:
344 + assert_never(inputs)
345 +
346 + return prompt, prompt_token_ids, multi_modal_data
347 +
348 + async def _process_encoder_decoder_prompt_async(
349 + self,
350 inputs: PromptInputs,
351 + request_id: str,
352 + ) -> EncoderDecoderLLMInputs:
353 + """Async version of :meth:`_process_encoder_decoder_prompt`."""
354 + encoder_comps: PromptComponents
355 + decoder_comps: DecoderPromptComponents
356 +
357 + if is_explicit_encoder_decoder_prompt(inputs):
358 + encoder_task = self._extract_prompt_components_async(
359 + inputs["encoder_prompt"],
360 + request_id=request_id,
361 + )
362 +
363 + if (decoder_input := inputs["decoder_prompt"]) is None:
364 + encoder_comps = await encoder_task
365 + decoder_comps = None, None, None
366 + else:
367 + decoder_task = self._extract_prompt_components_async(
368 + decoder_input,
369 + request_id=request_id,
370 + )
371 +
372 + encoder_comps, decoder_comps = await asyncio.gather(
373 + encoder_task, decoder_task)
374 + else:
375 + encoder_comps = await self._extract_prompt_components_async(
376 + inputs,
377 + request_id=request_id,
378 + )
379 +
380 + decoder_comps = None, None, None
381 +
382 + return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
383 +
384 + async def _process_decoder_only_prompt_async(
385 + self,
386 + inputs: SingletonPromptInputs,
387 + request_id: str,
388 lora_request: Optional[LoRARequest] = None,
389 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
390 ) -> LLMInputs:
303 - if isinstance(inputs, str):
304 - inputs = {"prompt": inputs}
391 + """Async version of :meth:`_process_decoder_only_prompt`."""
392 + prompt_comps = await self._extract_prompt_components_async(
393 + inputs,
394 + request_id=request_id,
395 + lora_request=lora_request,
396 + )
397
306 - if "prompt_token_ids" not in inputs:
307 - tokenizer = self.get_tokenizer_group("prompts must be None if "
308 - "skip_tokenizer_init is True")
398 + return self._build_decoder_only_llm_inputs(
399 + prompt_comps,
400 + prompt_adapter_request=prompt_adapter_request,
401 + )
402
310 - prompt_token_ids = await tokenizer.encode_async(
403 + async def process_model_inputs_async(
404 + self,
405 + inputs: PromptInputs,
406 + request_id: str,
407 + lora_request: Optional[LoRARequest] = None,
408 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
409 + ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
410 + """Async version of :meth:`process_model_inputs`."""
411 + if self.is_encoder_decoder_model():
412 + # Encoder-decoder model requires special mapping of
413 + # input prompts to encoder & decoder
414 + model_inputs = await self._process_encoder_decoder_prompt_async(
415 + inputs,
416 request_id=request_id,
312 - prompt=inputs["prompt"],
313 - lora_request=lora_request)
417 + )
418 else:
315 - prompt_token_ids = inputs["prompt_token_ids"]
419 + if is_explicit_encoder_decoder_prompt(inputs):
420 + raise ValueError("Cannot pass encoder-decoder prompt "
421 + "to decoder-only models")
422
317 - if prompt_adapter_request:
318 - prompt_token_ids = [
319 - 0
320 - ] * prompt_adapter_request.prompt_adapter_num_virtual_tokens + \
321 - prompt_token_ids
322 -
323 - llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
324 - prompt=inputs.get("prompt"),
325 - multi_modal_data=inputs.get("multi_modal_data"))
423 + # Decoder-only operation
424 + model_inputs = await self._process_decoder_only_prompt_async(
425 + inputs,
426 + request_id=request_id,
427 + lora_request=lora_request,
428 + prompt_adapter_request=prompt_adapter_request,
429 + )
430
327 - return self.input_processor(llm_inputs)
431 + return self.input_processor(model_inputs)
432
433 async def add_request_async(
434 self,
@@ -336,6 +440,7 @@ class _AsyncLLMEngine(LLMEngine):
440 trace_headers: Optional[Mapping[str, str]] = None,
441 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
442 ) -> None:
443 + """Async version of :meth:`add_request`."""
444 if lora_request is not None and not self.lora_config:
445 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
446 "not enabled!")
@@ -343,10 +448,11 @@ class _AsyncLLMEngine(LLMEngine):
448 arrival_time = time.time()
449
450 processed_inputs = await self.process_model_inputs_async(
451 + inputs,
452 request_id=request_id,
347 - inputs=inputs,
453 lora_request=lora_request,
349 - prompt_adapter_request=prompt_adapter_request)
454 + prompt_adapter_request=prompt_adapter_request,
455 + )
456
457 self._add_processed_request(
458 request_id=request_id,
vllm/engine/llm_engine.py
+151 -171
@@ -5,6 +5,8 @@ from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5 from typing import Sequence as GenericSequence
6 from typing import Set, Tuple, Type, TypeVar, Union
7
8 +from typing_extensions import assert_never
9 +
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -22,10 +24,12 @@ from vllm.engine.output_processor.stop_checker import StopChecker
24 from vllm.engine.output_processor.util import create_output_by_sequence_group
25 from vllm.executor.executor_base import ExecutorBase
26 from vllm.executor.ray_utils import initialize_ray_cluster
25 -from vllm.inputs import (INPUT_REGISTRY, LLMInputs, PromptInputs,
26 - get_prompt_type)
27 +from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs, LLMInputs,
28 + PromptInputs, SingletonPromptInputs)
29 +from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
30 from vllm.logger import init_logger
31 from vllm.lora.request import LoRARequest
32 +from vllm.multimodal import MultiModalDataDict
33 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
34 RequestOutputFactory)
35 from vllm.pooling_params import PoolingParams
@@ -43,8 +47,7 @@ from vllm.transformers_utils.tokenizer_group import (
47 AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
48 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
49 usage_message)
46 -from vllm.utils import (Counter, is_embedding_model_config,
47 - is_encoder_decoder_model_config)
50 +from vllm.utils import Counter
51 from vllm.version import __version__ as VLLM_VERSION
52
53 logger = init_logger(__name__)
@@ -66,6 +69,11 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
69
70 _O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
71
72 +PromptComponents = Tuple[Optional[str], List[int],
73 + Optional[MultiModalDataDict]]
74 +DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
75 + Optional[MultiModalDataDict]]
76 +
77
78 class LLMEngine:
79 """An LLM engine that receives requests and generates texts.
@@ -524,7 +532,7 @@ class LLMEngine:
532
533 return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
534
527 - def _get_decoder_start_token_id(self, ) -> Optional[int]:
535 + def _get_decoder_start_token_id(self) -> Optional[int]:
536 '''
537 Obtain the decoder start token id employed by an encoder/decoder
538 model. Returns None for non-encoder/decoder models or if the
@@ -553,7 +561,7 @@ class LLMEngine:
561 def _add_processed_request(
562 self,
563 request_id: str,
556 - processed_inputs: LLMInputs,
564 + processed_inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
565 params: Union[SamplingParams, PoolingParams],
566 arrival_time: float,
567 lora_request: Optional[LoRARequest],
@@ -613,11 +621,11 @@ class LLMEngine:
621 def stop_remote_worker_execution_loop(self) -> None:
622 self.model_executor.stop_remote_worker_execution_loop()
623
616 - _LLMInputComponentsType = Tuple[str, List[int], ]
624 + _LLMInputComponentsType = Tuple[str, List[int]]
625
626 def _prepare_decoder_input_ids_for_generation(
627 self,
620 - decoder_input_ids: Optional[List[int]] = None,
628 + decoder_input_ids: Optional[List[int]],
629 ) -> List[int]:
630 """
631 Prepares `decoder_input_ids` for generation with encoder-decoder models.
@@ -639,14 +647,13 @@ class LLMEngine:
647 * Processed token list
648 """
649
642 - decoder_start_token_id: Optional[int] = (
643 - self._get_decoder_start_token_id())
650 + decoder_start_token_id = self._get_decoder_start_token_id()
651 assert decoder_start_token_id is not None
652
653 if decoder_input_ids is None:
654 # no decoder prompt input ->
655 # use decoder_start_token_id as decoder_input_ids
649 - (decoder_input_ids) = self._get_default_enc_dec_decoder_prompt()
656 + decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
657
658 if (len(decoder_input_ids) == 0
659 or decoder_input_ids[0] != decoder_start_token_id):
@@ -657,12 +664,11 @@ class LLMEngine:
664 def _tokenize_prompt(
665 self,
666 prompt: str,
660 - request_id: Optional[str] = None,
661 - lora_request: Optional[str] = None,
667 + request_id: str,
668 + lora_request: Optional[LoRARequest],
669 ) -> List[int]:
670 '''
664 - Wrapper around application of the model's
665 - tokenizer.
671 + Wrapper around application of the model's tokenizer.
672
673 Arguments:
674
@@ -678,87 +684,72 @@ class LLMEngine:
684 tokenizer = self.get_tokenizer_group("prompts must be None if "
685 "skip_tokenizer_init is True")
686
681 - prompt_token_ids = tokenizer.encode(request_id=request_id,
682 - prompt=prompt,
683 - lora_request=lora_request)
684 -
685 - return prompt_token_ids
687 + return tokenizer.encode(request_id=request_id,
688 + prompt=prompt,
689 + lora_request=lora_request)
690
687 - def _extract_single_prompt_for_enc_dec_input(
691 + def _extract_prompt_components(
692 self,
689 - inputs: Optional[PromptInputs],
690 - request_id: Optional[str] = None,
691 - ptype: Optional[str] = None,
692 - is_encoder_prompt: bool = False,
693 - ) -> Tuple[Optional[str], List[int]]:
693 + inputs: SingletonPromptInputs,
694 + request_id: str,
695 + lora_request: Optional[LoRARequest] = None,
696 + ) -> PromptComponents:
697 '''
695 - Only for encoder/decoder models:
696 - Extract prompt & prompt_token_ids from any single
697 - encoder or decoder input prompt. For encoder input prompts
698 - in particular, also extract multi-modal data.
699 -
700 - This function handles the following scenarios:
701 - 1. The user supplied a singleton encoder prompt
702 - & the prompt/prompt-token-ids must be extracted.
703 - 2. The user supplied an explicit encoder/decoder
704 - prompt & the prompt/prompt-token-ids must be
705 - extracted from either the encoder and decoder prompts.
706 -
707 - For decoder prompts in particular (scenario 2), special
708 - processing is applied to the returned decoder token ids.
698 + Extract the components of any single encoder or decoder input prompt.
699
700 Arguments:
701
702 * request_id
713 - * ptype: str representation of the input prompt type.
714 - If `ptype` is `None`, assume that the prompt
715 - type is unknown and must be inferred. This is the
716 - case for ExplicitEncoderDecoder sub-prompts.
703 * inputs: single encoder or decoder input prompt
718 - * is_encoder_prompt: True if encoder input prompt.
719 - If False, decoder prompt tokens
720 - are preprocessed.
704 + * lora_request: this is only valid for decoder prompts
705
706 Returns:
707
708 * prompt
709 * prompt_token_ids
710 + * multi_modal_data
711 '''
727 - prompt_token_ids = None
728 - ptype = (get_prompt_type(inputs) if ptype is None else ptype)
712
730 - if inputs is None:
731 - prompt = None
732 - elif ptype == 'str':
713 + if isinstance(inputs, str):
714 prompt = inputs
715 prompt_token_ids = self._tokenize_prompt(
716 prompt,
717 request_id=request_id,
718 + lora_request=lora_request,
719 )
738 - elif ptype == 'TokensPrompt':
739 - prompt = None
740 - prompt_token_ids = inputs['prompt_token_ids']
720 + multi_modal_data = None
721 + elif isinstance(inputs, dict):
722 + if "prompt_token_ids" in inputs:
723 + prompt = None
724 + prompt_token_ids = inputs["prompt_token_ids"]
725 + else:
726 + # NOTE: This extra assignment is required to pass mypy
727 + prompt = parsed_prompt = inputs["prompt"]
728 + prompt_token_ids = self._tokenize_prompt(
729 + parsed_prompt,
730 + request_id=request_id,
731 + lora_request=lora_request,
732 + )
733 +
734 + multi_modal_data = inputs.get("multi_modal_data")
735 else:
742 - prompt = inputs['prompt']
743 - prompt_token_ids = self._tokenize_prompt(
744 - prompt,
745 - request_id=request_id,
746 - )
736 + assert_never(inputs)
737
748 - if not is_encoder_prompt:
749 - # Apply special pre-processing to
750 - # decoder prompts
751 - prompt_token_ids = (self._prepare_decoder_input_ids_for_generation(
752 - prompt_token_ids, ))
738 + return prompt, prompt_token_ids, multi_modal_data
739
754 - assert prompt_token_ids is not None
740 + def _apply_prompt_adapter(
741 + self,
742 + prompt_token_ids: List[int],
743 + prompt_adapter_request: Optional[PromptAdapterRequest],
744 + ) -> List[int]:
745 + if prompt_adapter_request:
746 + prompt_token_ids = (
747 + [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
748 + + prompt_token_ids)
749
756 - return (
757 - prompt,
758 - prompt_token_ids,
759 - )
750 + return prompt_token_ids
751
761 - def _get_default_enc_dec_decoder_prompt(self, ) -> List[int]:
752 + def _get_default_enc_dec_decoder_prompt(self) -> List[int]:
753 '''
754 Specifically for encoder/decoder models:
755 generate a default decoder prompt for when
@@ -792,18 +783,39 @@ class LLMEngine:
783
784 bos_token_id = self._get_bos_token_id()
785 assert bos_token_id is not None
795 - prompt_token_ids: List[int] = [bos_token_id]
796 - return prompt_token_ids
786 + return [bos_token_id]
787 +
788 + def _build_enc_dec_llm_inputs(
789 + self,
790 + encoder_comps: PromptComponents,
791 + decoder_comps: DecoderPromptComponents,
792 + ) -> EncoderDecoderLLMInputs:
793 + encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
794 + decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
795 +
796 + if encoder_mm_data is not None or decoder_mm_data is not None:
797 + raise ValueError("Multi-modal encoder-decoder models are "
798 + "not supported yet")
799 +
800 + decoder_prompt_ids = (
801 + self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
802 +
803 + return EncoderDecoderLLMInputs(
804 + prompt_token_ids=decoder_prompt_ids,
805 + prompt=decoder_prompt,
806 + encoder_prompt_token_ids=encoder_prompt_ids,
807 + encoder_prompt=encoder_prompt,
808 + )
809
810 def _process_encoder_decoder_prompt(
811 self,
812 inputs: PromptInputs,
801 - request_id: Optional[str] = None,
802 - ) -> LLMInputs:
813 + request_id: str,
814 + ) -> EncoderDecoderLLMInputs:
815 '''
816 For encoder/decoder models only:
805 - Process an input prompt
806 - into an `LLMInputs` instance.
817 + Process an input prompt into an
818 + :class:`EncoderDecoderLLMInputs` instance.
819
820 There are two types of input prompts:
821 singleton prompts which carry only the
@@ -830,136 +842,103 @@ class LLMEngine:
842
843 Returns:
844
833 - * `LLMInputs` instance
845 + * :class:`EncoderDecoderLLMInputs` instance
846 '''
847
836 - ptype = get_prompt_type(inputs)
837 -
838 - # Obtain encoder and decoder prompt tokens. Note
839 - # that, no matter what, the decoder
840 - # prompt type is unknown.
841 - if ptype == "ExplicitEncoderDecoder":
842 - # If input is explicit encoder/decoder prompt,
843 - # then it remains to be determined what type
844 - # of encoder prompt we have
845 - extracted_encoder_prompt = inputs.get('encoder_prompt')
846 - encoder_ptype = None
847 - # Extract decoder prompt from explicit
848 - # encoder/decoder prompt
849 - extracted_decoder_prompt = inputs.get('decoder_prompt')
848 + encoder_comps: PromptComponents
849 + decoder_comps: DecoderPromptComponents
850 +
851 + if is_explicit_encoder_decoder_prompt(inputs):
852 + encoder_comps = self._extract_prompt_components(
853 + inputs["encoder_prompt"],
854 + request_id=request_id,
855 + )
856 +
857 + if (decoder_input := inputs["decoder_prompt"]) is None:
858 + decoder_comps = None, None, None
859 + else:
860 + decoder_comps = self._extract_prompt_components(
861 + decoder_input,
862 + request_id=request_id,
863 + )
864 else:
851 - # If input is singleton encoder prompt, then
852 - # we know the encoder prompt type
853 - extracted_encoder_prompt = inputs
854 - encoder_ptype = ptype
855 - # Decoder prompt is always unknown if
856 - # encoder/decoder prompt is not explicit
857 - extracted_decoder_prompt = None
858 -
859 - # Invoke helper function to obtain encoder
860 - # prompt and prompt token ids, either from
861 - # singleton encoder prompt or from the
862 - # encoder sub-prompt of an explicit
863 - # encoder/decode scenario 2), special
864 - # processing is applied to the returned decoder token ids
865 - (
866 - encoder_prompt,
867 - encoder_prompt_token_ids,
868 - ) = self._extract_single_prompt_for_enc_dec_input(
869 - extracted_encoder_prompt,
870 - request_id=request_id,
871 - ptype=encoder_ptype,
872 - is_encoder_prompt=True,
873 - )
865 + encoder_comps = self._extract_prompt_components(
866 + inputs,
867 + request_id=request_id,
868 + )
869
875 - # Invoke helper method to obtain
876 - # decoder prompt and prompt token ids.
877 - #
878 - # The helper method will detect the decoder
879 - # prompt type.
880 - #
881 - # Helper method will also apply special
882 - # preprocessing unique to decoder prompts.
883 - (
884 - decoder_prompt,
885 - decoder_prompt_token_ids,
886 - ) = self._extract_single_prompt_for_enc_dec_input(
887 - extracted_decoder_prompt,
888 - request_id=request_id,
889 - ptype=None,
890 - is_encoder_prompt=False,
891 - )
870 + decoder_comps = None, None, None
871
893 - return LLMInputs(
894 - prompt_token_ids=decoder_prompt_token_ids,
895 - prompt=decoder_prompt,
896 - encoder_prompt_token_ids=encoder_prompt_token_ids,
897 - encoder_prompt=encoder_prompt,
898 - )
872 + return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
873 +
874 + def _build_decoder_only_llm_inputs(
875 + self,
876 + prompt_comps: PromptComponents,
877 + prompt_adapter_request: Optional[PromptAdapterRequest],
878 + ) -> LLMInputs:
879 + prompt, prompt_token_ids, multi_modal_data = prompt_comps
880 +
881 + prompt_token_ids = self._apply_prompt_adapter(
882 + prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
883 +
884 + return LLMInputs(prompt_token_ids=prompt_token_ids,
885 + prompt=prompt,
886 + multi_modal_data=multi_modal_data)
887
888 def _process_decoder_only_prompt(
889 self,
902 - inputs: PromptInputs,
890 + inputs: SingletonPromptInputs,
891 + request_id: str,
892 lora_request: Optional[LoRARequest] = None,
904 - request_id: Optional[str] = None,
893 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
894 ) -> LLMInputs:
895 '''
896 For decoder-only models:
909 - Process an input prompt
910 - into an `LLMInputs` instance.
897 + Process an input prompt into an :class:`LLMInputs` instance.
898
899 Arguments:
900
901 * inputs: input prompt
915 - * lora_request
902 * request_id
903 + * lora_request
904 * prompt_adapter_request
905
906 Returns:
907
921 - * `LLMInputs` instance
908 + * :class:`LLMInputs` instance
909 '''
910
924 - if isinstance(inputs, str):
925 - inputs = {"prompt": inputs}
926 - prompt = inputs.get("prompt")
927 -
928 - if "prompt_token_ids" not in inputs:
929 - prompt_token_ids = self._tokenize_prompt(
930 - prompt,
931 - request_id=request_id,
932 - lora_request=lora_request,
933 - )
934 - else:
935 - prompt_token_ids = inputs["prompt_token_ids"]
936 -
937 - if prompt_adapter_request:
938 - prompt_token_ids = (
939 - [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
940 - + prompt_token_ids)
911 + prompt_comps = self._extract_prompt_components(
912 + inputs,
913 + request_id=request_id,
914 + lora_request=lora_request,
915 + )
916
942 - return LLMInputs(prompt_token_ids=prompt_token_ids,
943 - prompt=prompt,
944 - multi_modal_data=inputs.get("multi_modal_data"))
917 + return self._build_decoder_only_llm_inputs(
918 + prompt_comps,
919 + prompt_adapter_request=prompt_adapter_request,
920 + )
921
922 def process_model_inputs(
923 self,
948 - request_id: str,
924 inputs: PromptInputs,
925 + request_id: str,
926 lora_request: Optional[LoRARequest] = None,
927 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
952 - ) -> LLMInputs:
928 + ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
929
930 if self.is_encoder_decoder_model():
931 # Encoder-decoder model requires special mapping of
932 # input prompts to encoder & decoder
957 -
933 model_inputs = self._process_encoder_decoder_prompt(
934 inputs,
935 request_id=request_id,
936 )
937 else:
938 + if is_explicit_encoder_decoder_prompt(inputs):
939 + raise ValueError("Cannot pass encoder-decoder prompt "
940 + "to decoder-only models")
941 +
942 # Decoder-only operation
943 model_inputs = self._process_decoder_only_prompt(
944 inputs,
@@ -1029,10 +1008,11 @@ class LLMEngine:
1008 arrival_time = time.time()
1009
1010 processed_inputs = self.process_model_inputs(
1011 + inputs,
1012 request_id=request_id,
1033 - inputs=inputs,
1013 lora_request=lora_request,
1035 - prompt_adapter_request=prompt_adapter_request)
1014 + prompt_adapter_request=prompt_adapter_request,
1015 + )
1016
1017 self._add_processed_request(
1018 request_id=request_id,
@@ -1597,7 +1577,7 @@ class LLMEngine:
1577 seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
1578
1579 def is_encoder_decoder_model(self):
1600 - return is_encoder_decoder_model_config(self.model_config)
1580 + return self.model_config.is_encoder_decoder_model
1581
1582 def is_embedding_model(self):
1603 - return is_embedding_model_config(self.model_config)
1583 + return self.model_config.is_embedding_model
vllm/entrypoints/chat_utils.py
+2 -3
@@ -2,8 +2,7 @@ import codecs
2 from dataclasses import dataclass
3 from functools import lru_cache
4 from pathlib import Path
5 -from typing import (Any, Awaitable, Iterable, List, Optional, Tuple, Union,
6 - cast, final)
5 +from typing import Any, Awaitable, Iterable, List, Optional, Tuple, Union, cast
6
7 # yapf conflicts with isort for this block
8 # yapf: disable
@@ -59,7 +58,7 @@ ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
58 CustomChatCompletionMessageParam]
59
60
62 -@final # So that it should be compatible with Dict[str, str]
61 +# TODO: Make fields ReadOnly once mypy supports it
62 class ConversationMessage(TypedDict):
63 role: str
64 content: str
vllm/entrypoints/llm.py
+2 -2
@@ -6,8 +6,8 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.llm_engine import LLMEngine
9 -from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt,
10 - parse_and_batch_prompt)
9 +from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
10 +from vllm.inputs.parse import parse_and_batch_prompt
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.model_executor.guided_decoding import (
vllm/entrypoints/openai/logits_processors.py
+5 -3
@@ -40,9 +40,11 @@ def _get_allowed_token_ids_logits_processor(
40 return AllowedTokenIdsLogitsProcessor(allowed_token_ids)
41
42
43 -def logit_bias_logits_processor(logit_bias: Dict[str,
44 - float], token_ids: List[int],
45 - logits: torch.Tensor) -> torch.Tensor:
43 +def logit_bias_logits_processor(
44 + logit_bias: Dict[int, float],
45 + token_ids: List[int],
46 + logits: torch.Tensor,
47 +) -> torch.Tensor:
48 for token_id, bias in logit_bias.items():
49 logits[token_id] += bias
50 return logits
vllm/entrypoints/openai/serving_engine.py
+1 -1
@@ -22,7 +22,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
22 TokenizeCompletionRequest,
23 TokenizeRequest)
24 # yapf: enable
25 -from vllm.inputs import parse_and_batch_prompt
25 +from vllm.inputs.parse import parse_and_batch_prompt
26 from vllm.logger import init_logger
27 from vllm.lora.request import LoRARequest
28 from vllm.model_executor.guided_decoding import (
vllm/inputs/__init__.py
+10 -11
@@ -1,7 +1,7 @@
1 -from .data import (ExplicitEncoderDecoderPrompt, LLMInputs, ParsedText,
2 - ParsedTokens, PromptInputs, SingletonPromptInputs,
3 - TextPrompt, TokensPrompt, get_prompt_type,
4 - is_valid_encoder_decoder_llm_inputs, parse_and_batch_prompt)
1 +from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2 + LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
3 + TokensPrompt, build_explicit_enc_dec_prompt,
4 + to_enc_dec_tuple_list, zip_enc_dec_prompts)
5 from .registry import InputContext, InputRegistry
6
7 INPUT_REGISTRY = InputRegistry()
@@ -14,18 +14,17 @@ See also:
14 """
15
16 __all__ = [
17 - "ParsedText",
18 - "ParsedTokens",
19 - "parse_and_batch_prompt",
17 "TextPrompt",
18 "TokensPrompt",
19 "PromptInputs",
20 + "SingletonPromptInputs",
21 + "ExplicitEncoderDecoderPrompt",
22 "LLMInputs",
23 + "EncoderDecoderLLMInputs",
24 + "build_explicit_enc_dec_prompt",
25 + "to_enc_dec_tuple_list",
26 + "zip_enc_dec_prompts",
27 "INPUT_REGISTRY",
28 "InputContext",
29 "InputRegistry",
27 - "get_prompt_type",
28 - "is_valid_encoder_decoder_llm_inputs",
29 - "ExplicitEncoderDecoderPrompt",
30 - "SingletonPromptInputs",
30 ]
vllm/inputs/data.py
+72 -135
@@ -1,71 +1,12 @@
1 -from typing import (TYPE_CHECKING, List, Literal, Optional, Sequence,
2 - TypedDict, Union, cast, overload)
1 +from typing import (TYPE_CHECKING, Generic, Iterable, List, Optional, Tuple,
2 + Union)
3
4 -from typing_extensions import NotRequired
4 +from typing_extensions import NotRequired, TypedDict, TypeVar
5
6 if TYPE_CHECKING:
7 from vllm.multimodal import MultiModalDataDict
8
9
10 -class ParsedText(TypedDict):
11 - content: str
12 - is_tokens: Literal[False]
13 -
14 -
15 -class ParsedTokens(TypedDict):
16 - content: List[int]
17 - is_tokens: Literal[True]
18 -
19 -
20 -# https://github.com/vllm-project/vllm/pull/4028
21 -@overload
22 -def parse_and_batch_prompt(
23 - prompt: Union[str, List[str]]) -> Sequence[ParsedText]:
24 - ...
25 -
26 -
27 -@overload
28 -def parse_and_batch_prompt(
29 - prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]:
30 - ...
31 -
32 -
33 -def parse_and_batch_prompt(
34 - prompt: Union[str, List[str], List[int], List[List[int]]],
35 -) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
36 - if isinstance(prompt, str):
37 - # case 1: a string
38 - return [ParsedText(content=prompt, is_tokens=False)]
39 -
40 - if isinstance(prompt, list):
41 - if len(prompt) == 0:
42 - raise ValueError("please provide at least one prompt")
43 -
44 - if isinstance(prompt[0], str):
45 - # case 2: array of strings
46 - return [
47 - ParsedText(content=elem, is_tokens=False)
48 - for elem in cast(List[str], prompt)
49 - ]
50 - if isinstance(prompt[0], int):
51 - # case 3: array of tokens
52 - elem = cast(List[int], prompt)
53 - return [ParsedTokens(content=elem, is_tokens=True)]
54 - if isinstance(prompt[0], list):
55 - if len(prompt[0]) == 0:
56 - raise ValueError("please provide at least one prompt")
57 -
58 - if isinstance(prompt[0][0], int):
59 - # case 4: array of token arrays
60 - return [
61 - ParsedTokens(content=elem, is_tokens=True)
62 - for elem in cast(List[List[int]], prompt)
63 - ]
64 -
65 - raise ValueError("prompt must be a string, array of strings, "
66 - "array of tokens, or array of token arrays")
67 -
68 -
10 class TextPrompt(TypedDict):
11 """Schema for a text prompt."""
12
@@ -103,39 +44,49 @@ Note that "singleton" is as opposed to a data structure
44 which encapsulates multiple prompts, i.e. of the sort
45 which may be utilized for encoder/decoder models when
46 the user desires to express both the encoder & decoder
106 -prompts explicitly, i.e. ExplicitEncoderDecoderPrompt
47 +prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
108 -A prompt of type SingletonPromptInputs may be employed
49 +A prompt of type :class:`SingletonPromptInputs` may be employed
50 as (1) input to a decoder-only model, (2) input to
51 the encoder of an encoder/decoder model, in the scenario
52 where the decoder-prompt is not specified explicitly, or
53 (3) as a member of a larger data structure encapsulating
113 -more than one prompt, i.e. ExplicitEncoderDecoderPrompt
54 +more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55 """
56
57 +_T1_co = TypeVar("_T1_co",
58 + bound=SingletonPromptInputs,
59 + default=SingletonPromptInputs,
60 + covariant=True)
61 +_T2_co = TypeVar("_T2_co",
62 + bound=SingletonPromptInputs,
63 + default=SingletonPromptInputs,
64 + covariant=True)
65
117 -class ExplicitEncoderDecoderPrompt(TypedDict):
66 +
67 +# TODO: Make fields ReadOnly once mypy supports it
68 +class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
69 """Represents an encoder/decoder model input prompt,
70 comprising an explicit encoder prompt and a
71 decoder prompt.
72
73 The encoder and decoder prompts, respectively,
74 may formatted according to any of the
124 - SingletonPromptInputs schemas, and are not
75 + :class:`SingletonPromptInputs` schemas, and are not
76 required to have the same schema.
77
78 Only the encoder prompt may have multi-modal data.
79
129 - Note that an ExplicitEncoderDecoderPrompt may not
80 + Note that an :class:`ExplicitEncoderDecoderPrompt` may not
81 be used as an input to a decoder-only model,
82 and that the `encoder_prompt` and `decoder_prompt`
132 - fields of this data structure may not themselves
133 - must be SingletonPromptInputs instances.
83 + fields of this data structure themselves must be
84 + :class:`SingletonPromptInputs` instances.
85 """
86
136 - encoder_prompt: SingletonPromptInputs
87 + encoder_prompt: _T1_co
88
138 - decoder_prompt: SingletonPromptInputs
89 + decoder_prompt: Optional[_T2_co]
90
91
92 PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
@@ -150,60 +101,12 @@ both decoder-only and encoder/decoder input types:
101 """
102
103
153 -def _has_required_keys(
154 - d: dict,
155 - required_keys: set,
156 -) -> bool:
157 - return required_keys.issubset(d.keys())
158 -
159 -
160 -def get_prompt_type(prompt: Optional[PromptInputs]) -> Optional[str]:
161 - """
162 - Get the type-name of the prompt argument instance, given that
163 - isinstance() cannot apply to TypedDict subclasses directly.
164 - If the prompt is None, return 'None' as the type name.
165 -
166 - Arguments:
167 -
168 - * prompt: LLM input prompt or None
169 -
170 - Returns:
171 -
172 - * String representation of prompt type
173 - """
174 -
175 - if prompt is None:
176 - return 'None'
177 -
178 - required_keys_dict = {
179 - 'TextPrompt': {'prompt'},
180 - 'TokensPrompt': {'prompt_token_ids'},
181 - 'ExplicitEncoderDecoder': {'encoder_prompt', 'decoder_prompt'},
182 - }
183 -
184 - if isinstance(prompt, dict):
185 - for (ptype, required_keys) in required_keys_dict.items():
186 - # Ignore type checking in the conditional below because type
187 - # checker does not understand that is_dict(prompt) narrows
188 - # down the possible types
189 - if _has_required_keys(
190 - prompt, # type: ignore
191 - required_keys):
192 - return ptype
193 -
194 - raise ValueError(f"Invalid prompt {prompt}, valid types are "
195 - "required_keys_dict={required_keys_dict}")
196 -
197 - if isinstance(prompt, str):
198 - return "str"
199 -
200 - raise ValueError(f"Invalid prompt {prompt}")
201 -
202 -
104 class LLMInputs(TypedDict):
105 """
106 The inputs in :class:`~vllm.LLMEngine` before they are
107 passed to the model executor.
108 +
109 + This specifies the data required for decoder-only models.
110 """
111 prompt_token_ids: List[int]
112 """The token IDs of the prompt."""
@@ -213,7 +116,21 @@ class LLMInputs(TypedDict):
116 The original prompt text corresponding to the token IDs, if available.
117 """
118
216 - encoder_prompt_token_ids: NotRequired[List[int]]
119 + multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
120 + """
121 + Optional multi-modal data to pass to the model,
122 + if the model supports it.
123 + """
124 +
125 +
126 +class EncoderDecoderLLMInputs(LLMInputs):
127 + """
128 + The inputs in :class:`~vllm.LLMEngine` before they are
129 + passed to the model executor.
130 +
131 + This specifies the required data for encoder-decoder models.
132 + """
133 + encoder_prompt_token_ids: List[int]
134 """The token IDs of the encoder prompt."""
135
136 encoder_prompt: NotRequired[Optional[str]]
@@ -222,20 +139,40 @@ class LLMInputs(TypedDict):
139 available.
140 """
141
225 - multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
226 - """
227 - Optional multi-modal data to pass to the model,
228 - if the model supports it.
229 - """
142 +
143 +_T1 = TypeVar("_T1",
144 + bound=SingletonPromptInputs,
145 + default=SingletonPromptInputs)
146 +_T2 = TypeVar("_T2",
147 + bound=SingletonPromptInputs,
148 + default=SingletonPromptInputs)
149
150
232 -def is_valid_encoder_decoder_llm_inputs(inputs: LLMInputs) -> bool:
151 +def build_explicit_enc_dec_prompt(
152 + encoder_prompt: _T1,
153 + decoder_prompt: Optional[_T2],
154 +) -> ExplicitEncoderDecoderPrompt[_T1, _T2]:
155 + return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
156 + decoder_prompt=decoder_prompt)
157 +
158 +
159 +def zip_enc_dec_prompts(
160 + enc_prompts: Iterable[_T1],
161 + dec_prompts: Iterable[Optional[_T2]],
162 +) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
163 """
234 - Return True if the LLMInputs instance has the correct configuration
235 - for encoder/decoder.
164 + Zip encoder and decoder prompts together into a list of
165 + :class:`ExplicitEncoderDecoderPrompt` instances.
166 """
167 + return [
168 + build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
169 + for (encoder_prompt, decoder_prompt) in zip(enc_prompts, dec_prompts)
170 + ]
171 +
172
238 - # True if encoder prompt token ids field exists &
239 - # is not None
240 - return ('encoder_prompt_token_ids' in inputs
241 - and inputs['encoder_prompt_token_ids'] is not None)
173 +def to_enc_dec_tuple_list(
174 + enc_dec_prompts: Iterable[ExplicitEncoderDecoderPrompt[_T1, _T2]],
175 +) -> List[Tuple[_T1, Optional[_T2]]]:
176 + return [(enc_dec_prompt["encoder_prompt"],
177 + enc_dec_prompt["decoder_prompt"])
178 + for enc_dec_prompt in enc_dec_prompts]
vllm/inputs/parse.py new
+75
@@ -0,0 +1,75 @@
1 +from typing import List, Literal, Sequence, TypedDict, Union, overload
2 +
3 +from typing_extensions import TypeIs
4 +
5 +from vllm.utils import is_list_of
6 +
7 +from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8 + LLMInputs, PromptInputs)
9 +
10 +
11 +class ParsedText(TypedDict):
12 + content: str
13 + is_tokens: Literal[False]
14 +
15 +
16 +class ParsedTokens(TypedDict):
17 + content: List[int]
18 + is_tokens: Literal[True]
19 +
20 +
21 +@overload
22 +def parse_and_batch_prompt(
23 + prompt: Union[str, List[str]]) -> Sequence[ParsedText]:
24 + ...
25 +
26 +
27 +@overload
28 +def parse_and_batch_prompt(
29 + prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]:
30 + ...
31 +
32 +
33 +def parse_and_batch_prompt(
34 + prompt: Union[str, List[str], List[int], List[List[int]]],
35 +) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
36 + if isinstance(prompt, str):
37 + # case 1: a string
38 + return [ParsedText(content=prompt, is_tokens=False)]
39 +
40 + if isinstance(prompt, list):
41 + if len(prompt) == 0:
42 + raise ValueError("please provide at least one prompt")
43 +
44 + if is_list_of(prompt, str):
45 + # case 2: array of strings
46 + return [
47 + ParsedText(content=elem, is_tokens=False) for elem in prompt
48 + ]
49 + if is_list_of(prompt, int):
50 + # case 3: array of tokens
51 + return [ParsedTokens(content=prompt, is_tokens=True)]
52 + if is_list_of(prompt, list):
53 + if len(prompt[0]) == 0:
54 + raise ValueError("please provide at least one prompt")
55 +
56 + if is_list_of(prompt[0], int):
57 + # case 4: array of token arrays
58 + return [
59 + ParsedTokens(content=elem, is_tokens=True)
60 + for elem in prompt
61 + ]
62 +
63 + raise ValueError("prompt must be a string, array of strings, "
64 + "array of tokens, or array of token arrays")
65 +
66 +
67 +def is_explicit_encoder_decoder_prompt(
68 + inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
69 + return isinstance(inputs, dict) and "encoder_prompt" in inputs
70 +
71 +
72 +def is_valid_encoder_decoder_llm_inputs(
73 + inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
74 +) -> TypeIs[EncoderDecoderLLMInputs]:
75 + return "encoder_prompt_token_ids" in inputs
vllm/model_executor/models/interfaces.py
+11 -11
@@ -1,7 +1,7 @@
1 from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
2 Union, overload, runtime_checkable)
3
4 -from typing_extensions import TypeGuard
4 +from typing_extensions import TypeIs
5
6 from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
7 from vllm.logger import init_logger
@@ -37,18 +37,18 @@ class _SupportsVisionType(Protocol):
37
38
39 @overload
40 -def supports_vision(model: Type[object]) -> TypeGuard[Type[SupportsVision]]:
40 +def supports_vision(model: Type[object]) -> TypeIs[Type[SupportsVision]]:
41 ...
42
43
44 @overload
45 -def supports_vision(model: object) -> TypeGuard[SupportsVision]:
45 +def supports_vision(model: object) -> TypeIs[SupportsVision]:
46 ...
47
48
49 def supports_vision(
50 model: Union[Type[object], object],
51 -) -> Union[TypeGuard[Type[SupportsVision]], TypeGuard[SupportsVision]]:
51 +) -> Union[TypeIs[Type[SupportsVision]], TypeIs[SupportsVision]]:
52 if isinstance(model, type):
53 return isinstance(model, _SupportsVisionType)
54
@@ -94,18 +94,18 @@ class _SupportsLoRAType(Protocol):
94
95
96 @overload
97 -def supports_lora(model: Type[object]) -> TypeGuard[Type[SupportsLoRA]]:
97 +def supports_lora(model: Type[object]) -> TypeIs[Type[SupportsLoRA]]:
98 ...
99
100
101 @overload
102 -def supports_lora(model: object) -> TypeGuard[SupportsLoRA]:
102 +def supports_lora(model: object) -> TypeIs[SupportsLoRA]:
103 ...
104
105
106 def supports_lora(
107 model: Union[Type[object], object],
108 -) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
108 +) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
109 result = _supports_lora(model)
110
111 if not result:
@@ -137,7 +137,7 @@ def supports_lora(
137
138 def _supports_lora(
139 model: Union[Type[object], object],
140 -) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
140 +) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
141 if isinstance(model, type):
142 return isinstance(model, _SupportsLoRAType)
143
@@ -172,18 +172,18 @@ class _HasInnerStateType(Protocol):
172
173
174 @overload
175 -def has_inner_state(model: object) -> TypeGuard[HasInnerState]:
175 +def has_inner_state(model: object) -> TypeIs[HasInnerState]:
176 ...
177
178
179 @overload
180 -def has_inner_state(model: Type[object]) -> TypeGuard[Type[HasInnerState]]:
180 +def has_inner_state(model: Type[object]) -> TypeIs[Type[HasInnerState]]:
181 ...
182
183
184 def has_inner_state(
185 model: Union[Type[object], object]
186 -) -> Union[TypeGuard[Type[HasInnerState]], TypeGuard[HasInnerState]]:
186 +) -> Union[TypeIs[Type[HasInnerState]], TypeIs[HasInnerState]]:
187 if isinstance(model, type):
188 return isinstance(model, _HasInnerStateType)
189
vllm/multimodal/image.py
+4 -2
@@ -10,6 +10,7 @@ from vllm.inputs.registry import InputContext
10 from vllm.logger import init_logger
11 from vllm.transformers_utils.image_processor import get_image_processor
12 from vllm.transformers_utils.tokenizer import get_tokenizer
13 +from vllm.utils import is_list_of
14
15 from .base import MultiModalInputs, MultiModalPlugin
16
@@ -113,7 +114,8 @@ class ImagePlugin(MultiModalPlugin):
114 def _default_input_mapper(self, ctx: InputContext,
115 data: object) -> MultiModalInputs:
116 model_config = ctx.model_config
116 - if isinstance(data, (Image.Image, list)):
117 +
118 + if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
119 image_processor = self._get_hf_image_processor(model_config)
120 if image_processor is None:
121 raise RuntimeError("No HuggingFace processor is available "
@@ -127,7 +129,7 @@ class ImagePlugin(MultiModalPlugin):
129 raise
130
131 return MultiModalInputs(batch_data)
130 - elif isinstance(data, torch.Tensor):
132 + elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor):
133 raise NotImplementedError("Embeddings input is not supported yet")
134
135 raise TypeError(f"Invalid image type: {type(data)}")
vllm/sequence.py
+1 -1
@@ -11,7 +11,7 @@ from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
11
12 import torch
13
14 -from vllm.inputs import is_valid_encoder_decoder_llm_inputs
14 +from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
15 from vllm.lora.request import LoRARequest
16 from vllm.pooling_params import PoolingParams
17 from vllm.prompt_adapter.request import PromptAdapterRequest
vllm/utils.py
+22 -52
@@ -17,8 +17,8 @@ from collections import defaultdict
17 from functools import lru_cache, partial, wraps
18 from platform import uname
19 from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
20 - Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
21 - Union, overload)
20 + Hashable, List, Literal, Optional, OrderedDict, Set, Tuple,
21 + Type, TypeVar, Union, overload)
22 from uuid import uuid4
23
24 import numpy as np
@@ -26,12 +26,10 @@ import numpy.typing as npt
26 import psutil
27 import torch
28 import torch.types
29 -from typing_extensions import ParamSpec
29 +from typing_extensions import ParamSpec, TypeIs, assert_never
30
31 import vllm.envs as envs
32 from vllm import _custom_ops as ops
33 -from vllm.inputs import (ExplicitEncoderDecoderPrompt, PromptInputs,
34 - SingletonPromptInputs)
33 from vllm.logger import enable_trace_function_call, init_logger
34
35 logger = init_logger(__name__)
@@ -812,6 +810,24 @@ def get_dtype_size(dtype: torch.dtype) -> int:
810 return torch.tensor([], dtype=dtype).element_size()
811
812
813 +# `collections` helpers
814 +def is_list_of(
815 + value: object,
816 + typ: Type[T],
817 + *,
818 + check: Literal["first", "all"] = "first",
819 +) -> TypeIs[List[T]]:
820 + if not isinstance(value, list):
821 + return False
822 +
823 + if check == "first":
824 + return len(value) == 0 or isinstance(value[0], typ)
825 + elif check == "all":
826 + return all(isinstance(v, typ) for v in value)
827 +
828 + assert_never(check)
829 +
830 +
831 def merge_dicts(dict1: Dict[K, List[T]],
832 dict2: Dict[K, List[T]]) -> Dict[K, List[T]]:
833 """Merge 2 dicts that have key -> List of items.
@@ -959,6 +975,7 @@ def enable_trace_function_call_for_thread() -> None:
975 enable_trace_function_call(log_path)
976
977
978 +# `functools` helpers
979 def identity(value: T) -> T:
980 return value
981
@@ -1080,50 +1097,3 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1097 """Utility function to run async task in a lock"""
1098 async with lock:
1099 return await task(*args, **kwargs)
1083 -
1084 -
1085 -def is_encoder_decoder_model_config(model_config) -> bool:
1086 - '''
1087 - Extract the HF encoder/decoder model flag from the ModelConfig instance.
1088 - Return False if model_config is None.
1089 - '''
1090 - return model_config is not None and \
1091 - getattr(model_config.hf_config,
1092 - "is_encoder_decoder",
1093 - False)
1094 -
1095 -
1096 -def is_embedding_model_config(model_config) -> bool:
1097 - '''
1098 - Extract the embedding model flag from the ModelConfig instance.
1099 - Return False if model_config is None.
1100 - '''
1101 - return model_config is not None and \
1102 - model_config.embedding_mode
1103 -
1104 -
1105 -def build_explicit_enc_dec_prompt(
1106 - encoder_prompt: SingletonPromptInputs,
1107 - decoder_prompt: SingletonPromptInputs,
1108 -) -> ExplicitEncoderDecoderPrompt:
1109 - return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
1110 - decoder_prompt=decoder_prompt)
1111 -
1112 -
1113 -def zip_enc_dec_prompt_lists(
1114 - enc_prompt_list: List[SingletonPromptInputs],
1115 - dec_prompt_list: List[SingletonPromptInputs],
1116 -) -> List[ExplicitEncoderDecoderPrompt]:
1117 - return [
1118 - build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
1119 - for (encoder_prompt,
1120 - decoder_prompt) in zip(enc_prompt_list, dec_prompt_list)
1121 - ]
1122 -
1123 -
1124 -def to_enc_dec_tuple_list(
1125 - enc_dec_prompts: List[ExplicitEncoderDecoderPrompt],
1126 -) -> List[Tuple[PromptInputs, PromptInputs]]:
1127 - return [(enc_dec_prompt['encoder_prompt'],
1128 - enc_dec_prompt['decoder_prompt'])
1129 - for enc_dec_prompt in enc_dec_prompts]
vllm/worker/worker.py
+2 -4
@@ -19,8 +19,6 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19 from vllm.platforms import current_platform
20 from vllm.prompt_adapter.request import PromptAdapterRequest
21 from vllm.sequence import ExecuteModelRequest
22 -from vllm.utils import (is_embedding_model_config,
23 - is_encoder_decoder_model_config)
22 from vllm.worker.cache_engine import CacheEngine
23 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
24 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
@@ -113,10 +111,10 @@ class Worker(LocalOrDistributedWorkerBase):
111 self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
112
113 def _is_encoder_decoder_model(self):
116 - return is_encoder_decoder_model_config(self.model_config)
114 + return self.model_config.is_encoder_decoder_model
115
116 def _is_embedding_model(self):
119 - return is_embedding_model_config(self.model_config)
117 + return self.model_config.is_embedding_model
118
119 def init_device(self) -> None:
120 if self.device_config.device.type == "cuda":