A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Support serving encoder/decoder models (#7258)
Cyrus Leung committed
Aug 9, 2024 at 10:39 UTC
7eb4a51c5f34a52427d170d0e654e0a346c6d69c
25 files changed
+603
-464
.github/workflows/mypy.yaml
+1
-1
@@ -25,7 +25,7 @@ jobs:
25
- name: Install dependencies
26
run: |
27
python -m pip install --upgrade pip
28
- pip install mypy==1.9.0
28
+ pip install mypy==1.11.1
29
pip install types-setuptools
30
pip install types-PyYAML
31
pip install types-requests
examples/offline_inference_encoder_decoder.py
+4
-4
@@ -4,8 +4,8 @@ encoder/decoder models, specifically BART
4
'''
5
6
from vllm import LLM, SamplingParams
7
-from vllm.inputs import ExplicitEncoderDecoderPrompt, TextPrompt, TokensPrompt
8
-from vllm.utils import zip_enc_dec_prompt_lists
7
+from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
8
+ TokensPrompt, zip_enc_dec_prompts)
9
10
dtype = "float"
11
@@ -61,9 +61,9 @@ enc_dec_prompt3 = ExplicitEncoderDecoderPrompt(
61
)
62
63
# - Finally, here's a useful helper function for zipping encoder and
64
-# decoder prompt lists together into a list of ExplicitEncoderDecoderPrompt
64
+# decoder prompts together into a list of ExplicitEncoderDecoderPrompt
65
# instances
66
-zipped_prompt_list = zip_enc_dec_prompt_lists(
66
+zipped_prompt_list = zip_enc_dec_prompts(
67
['An encoder prompt', 'Another encoder prompt'],
68
['A decoder prompt', 'Another decoder prompt'])
69
requirements-common.txt
+1
-1
@@ -19,7 +19,7 @@ prometheus-fastapi-instrumentator >= 7.0.0
19
tiktoken >= 0.6.0 # Required for DBRX tokenizer
20
lm-format-enforcer == 0.10.3
21
outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
22
-typing_extensions
22
+typing_extensions >= 4.10
23
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
24
pyzmq
25
gguf == 0.9.1
requirements-lint.txt
+1
-1
@@ -8,7 +8,7 @@ isort==5.13.2
8
clang-format==18.1.5
9
10
# type checking
11
-mypy==1.9.0
11
+mypy==1.11.1
12
types-PyYAML
13
types-requests
14
types-setuptools
tests/conftest.py
+19
-13
@@ -3,6 +3,7 @@ import gc
3
import os
4
import sys
5
from collections import UserList
6
+from enum import Enum
7
from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union
8
9
import pytest
@@ -14,20 +15,19 @@ from transformers import (AutoModelForCausalLM, AutoModelForSeq2SeqLM,
15
AutoModelForVision2Seq, AutoTokenizer, BatchEncoding,
16
BatchFeature)
17
17
-from tests.models.utils import DecoderPromptType
18
from vllm import LLM, SamplingParams
19
from vllm.assets.image import ImageAsset
20
from vllm.config import TokenizerPoolConfig
21
from vllm.connections import global_http_connection
22
from vllm.distributed import (destroy_distributed_environment,
23
destroy_model_parallel)
24
-from vllm.inputs import TextPrompt
24
+from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
25
+ to_enc_dec_tuple_list, zip_enc_dec_prompts)
26
from vllm.logger import init_logger
27
from vllm.outputs import RequestOutput
28
from vllm.sequence import SampleLogprobs
29
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
29
- is_cpu, to_enc_dec_tuple_list,
30
- zip_enc_dec_prompt_lists)
30
+ is_cpu)
31
32
logger = init_logger(__name__)
33
@@ -124,10 +124,16 @@ def example_prompts() -> List[str]:
124
return prompts
125
126
127
+class DecoderPromptType(Enum):
128
+ """For encoder/decoder models only."""
129
+ CUSTOM = 1
130
+ NONE = 2
131
+ EMPTY_STR = 3
132
+
133
+
134
@pytest.fixture
128
-def example_encoder_decoder_prompts() \
129
- -> Dict[DecoderPromptType,
130
- Tuple[List[str], List[Optional[str]]]]:
135
+def example_encoder_decoder_prompts(
136
+) -> Dict[DecoderPromptType, List[ExplicitEncoderDecoderPrompt]]:
137
'''
138
Returns an encoder prompt list and a decoder prompt list, wherein each pair
139
of same-index entries in both lists corresponds to an (encoder prompt,
@@ -150,11 +156,11 @@ def example_encoder_decoder_prompts() \
156
# NONE decoder prompt type
157
return {
158
DecoderPromptType.NONE:
153
- zip_enc_dec_prompt_lists(encoder_prompts, none_decoder_prompts),
159
+ zip_enc_dec_prompts(encoder_prompts, none_decoder_prompts),
160
DecoderPromptType.EMPTY_STR:
155
- zip_enc_dec_prompt_lists(encoder_prompts, empty_str_decoder_prompts),
161
+ zip_enc_dec_prompts(encoder_prompts, empty_str_decoder_prompts),
162
DecoderPromptType.CUSTOM:
157
- zip_enc_dec_prompt_lists(encoder_prompts, custom_decoder_prompts),
163
+ zip_enc_dec_prompts(encoder_prompts, custom_decoder_prompts),
164
}
165
166
@@ -444,7 +450,7 @@ class HfRunner:
450
451
def generate_encoder_decoder_greedy_logprobs_limit(
452
self,
447
- encoder_decoder_prompts: Tuple[List[str], List[str]],
453
+ encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
454
max_tokens: int,
455
num_logprobs: int,
456
**kwargs: Any,
@@ -608,7 +614,7 @@ class VllmRunner:
614
615
def generate_encoder_decoder_w_logprobs(
616
self,
611
- encoder_decoder_prompts: Tuple[List[str], List[str]],
617
+ encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
618
sampling_params: SamplingParams,
619
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
620
'''
@@ -653,7 +659,7 @@ class VllmRunner:
659
660
def generate_encoder_decoder_greedy_logprobs(
661
self,
656
- encoder_decoder_prompts: Tuple[List[str], List[str]],
662
+ encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
663
max_tokens: int,
664
num_logprobs: int,
665
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
tests/distributed/test_basic_distributed_correctness_enc_dec.py
+1
-1
@@ -11,9 +11,9 @@ pytest distributed/test_basic_distributed_correctness_enc_dec.py
11
12
import pytest
13
14
-from tests.models.utils import DecoderPromptType
14
from vllm.utils import cuda_device_count_stateless
15
16
+from ..conftest import DecoderPromptType
17
from ..models.utils import check_logprobs_close
18
from ..utils import fork_new_process_for_each_test
19
tests/entrypoints/openai/test_encoder_decoder.py
new
+50
@@ -0,0 +1,50 @@
1
+import openai
2
+import pytest
3
+
4
+from ...utils import RemoteOpenAIServer
5
+
6
+MODEL_NAME = "facebook/bart-base"
7
+
8
+
9
+@pytest.fixture(scope="module")
10
+def server():
11
+ args = [
12
+ "--dtype",
13
+ "bfloat16",
14
+ "--enforce-eager",
15
+ ]
16
+
17
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
18
+ yield remote_server
19
+
20
+
21
+@pytest.fixture(scope="module")
22
+def client(server):
23
+ return server.get_async_client()
24
+
25
+
26
+@pytest.mark.asyncio
27
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
28
+async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
29
+ completion = await client.completions.create(model=model_name,
30
+ prompt="Hello, my name is",
31
+ max_tokens=5,
32
+ temperature=0.0)
33
+
34
+ assert completion.id is not None
35
+ assert completion.choices is not None and len(completion.choices) == 1
36
+
37
+ choice = completion.choices[0]
38
+ assert len(choice.text) >= 5
39
+ assert choice.finish_reason == "length"
40
+ assert completion.usage == openai.types.CompletionUsage(
41
+ completion_tokens=5, prompt_tokens=2, total_tokens=7)
42
+
43
+ # test using token IDs
44
+ completion = await client.completions.create(
45
+ model=model_name,
46
+ prompt=[0, 0, 0, 0, 0],
47
+ max_tokens=5,
48
+ temperature=0.0,
49
+ )
50
+ assert len(completion.choices[0].text) >= 1
tests/models/test_bart.py
+27
-11
@@ -2,6 +2,8 @@
2
3
Run `pytest tests/models/test_bart.py`.
4
"""
5
+from typing import List, Optional, Tuple
6
+
7
from vllm.utils import is_cpu
8
9
if not is_cpu():
@@ -11,22 +13,31 @@ if not is_cpu():
13
14
import pytest
15
14
- from tests.models.utils import DecoderPromptType
16
+ from vllm.sequence import SampleLogprobs
17
18
+ from ..conftest import DecoderPromptType
19
from .utils import check_logprobs_close
20
21
MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
22
20
- DECODER_PROMPT_TYPES = ([
21
- DecoderPromptType.CUSTOM, DecoderPromptType.EMPTY_STR,
22
- DecoderPromptType.NONE
23
- ])
23
+ def vllm_to_hf_output(
24
+ vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
25
+ decoder_prompt_type: DecoderPromptType,
26
+ ):
27
+ """Sanitize vllm output to be comparable with hf output."""
28
+ output_ids, output_str, out_logprobs = vllm_output
29
+
30
+ hf_output_str = output_str + "</s>"
31
+ if decoder_prompt_type == DecoderPromptType.NONE:
32
+ hf_output_str = "<s>" + hf_output_str
33
+
34
+ return output_ids, hf_output_str, out_logprobs
35
36
@pytest.mark.parametrize("model", MODELS)
37
@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
38
@pytest.mark.parametrize("max_tokens", [64])
39
@pytest.mark.parametrize("num_logprobs", [5])
29
- @pytest.mark.parametrize("decoder_prompt_type", DECODER_PROMPT_TYPES)
40
+ @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
41
def test_models(
42
hf_runner,
43
vllm_runner,
@@ -146,8 +157,13 @@ if not is_cpu():
157
hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
158
else 0)
159
149
- check_logprobs_close(outputs_0_lst=hf_outputs,
150
- outputs_1_lst=vllm_outputs,
151
- name_0="hf",
152
- name_1="vllm",
153
- num_outputs_0_skip_tokens=hf_skip_tokens)
160
+ check_logprobs_close(
161
+ outputs_0_lst=hf_outputs,
162
+ outputs_1_lst=[
163
+ vllm_to_hf_output(vllm_output, decoder_prompt_type)
164
+ for vllm_output in vllm_outputs
165
+ ],
166
+ name_0="hf",
167
+ name_1="vllm",
168
+ num_outputs_0_skip_tokens=hf_skip_tokens,
169
+ )
tests/models/utils.py
-11
@@ -1,5 +1,4 @@
1
import warnings
2
-from enum import Enum
2
from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4
from vllm.sequence import SampleLogprobs
@@ -136,13 +135,3 @@ def check_logprobs_close(
135
warnings.simplefilter("always")
136
137
warnings.warn(fail_msg, stacklevel=2)
139
-
140
-
141
-class DecoderPromptType(Enum):
142
- '''
143
- For encoder/decoder models only -
144
-
145
- '''
146
- CUSTOM = 1
147
- NONE = 2
148
- EMPTY_STR = 3
tests/test_inputs.py
+1
-1
@@ -2,7 +2,7 @@ from typing import List
2
3
import pytest
4
5
-from vllm.inputs import parse_and_batch_prompt
5
+from vllm.inputs.parse import parse_and_batch_prompt
6
7
STRING_INPUTS = [
8
'',
vllm/config.py
+10
@@ -464,6 +464,16 @@ class ModelConfig:
464
if t != "attention"
465
])
466
467
+ @property
468
+ def is_encoder_decoder_model(self) -> bool:
469
+ """Extract the HF encoder/decoder model flag."""
470
+ return getattr(self.hf_config, "is_encoder_decoder", False)
471
+
472
+ @property
473
+ def is_embedding_model(self) -> bool:
474
+ """Extract the embedding model flag."""
475
+ return self.embedding_mode
476
+
477
478
class CacheConfig:
479
"""Configuration for the KV cache.
vllm/engine/async_llm_engine.py
+130
-24
@@ -5,6 +5,7 @@ from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
5
Optional, Set, Tuple, Type, Union)
6
7
from transformers import PreTrainedTokenizer
8
+from typing_extensions import assert_never
9
10
import vllm.envs as envs
11
from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
@@ -12,11 +13,14 @@ from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
13
from vllm.core.scheduler import SchedulerOutputs
14
from vllm.engine.arg_utils import AsyncEngineArgs
15
from vllm.engine.async_timeout import asyncio_timeout
15
-from vllm.engine.llm_engine import LLMEngine
16
+from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
17
+ PromptComponents)
18
from vllm.engine.metrics import StatLoggerBase
19
from vllm.executor.executor_base import ExecutorAsyncBase
20
from vllm.executor.ray_utils import initialize_ray_cluster, ray
19
-from vllm.inputs import LLMInputs, PromptInputs
21
+from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
22
+ SingletonPromptInputs)
23
+from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
24
from vllm.logger import init_logger
25
from vllm.lora.request import LoRARequest
26
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -293,38 +297,138 @@ class _AsyncLLMEngine(LLMEngine):
297
"""Stop the remote worker execution loop."""
298
await self.model_executor.stop_remote_worker_execution_loop_async()
299
296
- async def process_model_inputs_async(
300
+ async def _tokenize_prompt_async(
301
+ self,
302
+ prompt: str,
303
+ request_id: str,
304
+ lora_request: Optional[LoRARequest],
305
+ ) -> List[int]:
306
+ """Async version of :meth:`_tokenize_prompt`."""
307
+ tokenizer = self.get_tokenizer_group("prompts must be None if "
308
+ "skip_tokenizer_init is True")
309
+
310
+ return await tokenizer.encode_async(request_id=request_id,
311
+ prompt=prompt,
312
+ lora_request=lora_request)
313
+
314
+ async def _extract_prompt_components_async(
315
self,
316
+ inputs: SingletonPromptInputs,
317
request_id: str,
318
+ lora_request: Optional[LoRARequest] = None,
319
+ ) -> PromptComponents:
320
+ """Async version of :meth:`_extract_prompt_components`."""
321
+ if isinstance(inputs, str):
322
+ prompt = inputs
323
+ prompt_token_ids = await self._tokenize_prompt_async(
324
+ prompt,
325
+ request_id=request_id,
326
+ lora_request=lora_request,
327
+ )
328
+ multi_modal_data = None
329
+ elif isinstance(inputs, dict):
330
+ if "prompt_token_ids" in inputs:
331
+ prompt = None
332
+ prompt_token_ids = inputs["prompt_token_ids"]
333
+ else:
334
+ # NOTE: This extra assignment is required to pass mypy
335
+ prompt = parsed_prompt = inputs["prompt"]
336
+ prompt_token_ids = await self._tokenize_prompt_async(
337
+ parsed_prompt,
338
+ request_id=request_id,
339
+ lora_request=lora_request,
340
+ )
341
+
342
+ multi_modal_data = inputs.get("multi_modal_data")
343
+ else:
344
+ assert_never(inputs)
345
+
346
+ return prompt, prompt_token_ids, multi_modal_data
347
+
348
+ async def _process_encoder_decoder_prompt_async(
349
+ self,
350
inputs: PromptInputs,
351
+ request_id: str,
352
+ ) -> EncoderDecoderLLMInputs:
353
+ """Async version of :meth:`_process_encoder_decoder_prompt`."""
354
+ encoder_comps: PromptComponents
355
+ decoder_comps: DecoderPromptComponents
356
+
357
+ if is_explicit_encoder_decoder_prompt(inputs):
358
+ encoder_task = self._extract_prompt_components_async(
359
+ inputs["encoder_prompt"],
360
+ request_id=request_id,
361
+ )
362
+
363
+ if (decoder_input := inputs["decoder_prompt"]) is None:
364
+ encoder_comps = await encoder_task
365
+ decoder_comps = None, None, None
366
+ else:
367
+ decoder_task = self._extract_prompt_components_async(
368
+ decoder_input,
369
+ request_id=request_id,
370
+ )
371
+
372
+ encoder_comps, decoder_comps = await asyncio.gather(
373
+ encoder_task, decoder_task)
374
+ else:
375
+ encoder_comps = await self._extract_prompt_components_async(
376
+ inputs,
377
+ request_id=request_id,
378
+ )
379
+
380
+ decoder_comps = None, None, None
381
+
382
+ return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
383
+
384
+ async def _process_decoder_only_prompt_async(
385
+ self,
386
+ inputs: SingletonPromptInputs,
387
+ request_id: str,
388
lora_request: Optional[LoRARequest] = None,
389
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
390
) -> LLMInputs:
303
- if isinstance(inputs, str):
304
- inputs = {"prompt": inputs}
391
+ """Async version of :meth:`_process_decoder_only_prompt`."""
392
+ prompt_comps = await self._extract_prompt_components_async(
393
+ inputs,
394
+ request_id=request_id,
395
+ lora_request=lora_request,
396
+ )
397
306
- if "prompt_token_ids" not in inputs:
307
- tokenizer = self.get_tokenizer_group("prompts must be None if "
308
- "skip_tokenizer_init is True")
398
+ return self._build_decoder_only_llm_inputs(
399
+ prompt_comps,
400
+ prompt_adapter_request=prompt_adapter_request,
401
+ )
402
310
- prompt_token_ids = await tokenizer.encode_async(
403
+ async def process_model_inputs_async(
404
+ self,
405
+ inputs: PromptInputs,
406
+ request_id: str,
407
+ lora_request: Optional[LoRARequest] = None,
408
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
409
+ ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
410
+ """Async version of :meth:`process_model_inputs`."""
411
+ if self.is_encoder_decoder_model():
412
+ # Encoder-decoder model requires special mapping of
413
+ # input prompts to encoder & decoder
414
+ model_inputs = await self._process_encoder_decoder_prompt_async(
415
+ inputs,
416
request_id=request_id,
312
- prompt=inputs["prompt"],
313
- lora_request=lora_request)
417
+ )
418
else:
315
- prompt_token_ids = inputs["prompt_token_ids"]
419
+ if is_explicit_encoder_decoder_prompt(inputs):
420
+ raise ValueError("Cannot pass encoder-decoder prompt "
421
+ "to decoder-only models")
422
317
- if prompt_adapter_request:
318
- prompt_token_ids = [
319
- 0
320
- ] * prompt_adapter_request.prompt_adapter_num_virtual_tokens + \
321
- prompt_token_ids
322
-
323
- llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
324
- prompt=inputs.get("prompt"),
325
- multi_modal_data=inputs.get("multi_modal_data"))
423
+ # Decoder-only operation
424
+ model_inputs = await self._process_decoder_only_prompt_async(
425
+ inputs,
426
+ request_id=request_id,
427
+ lora_request=lora_request,
428
+ prompt_adapter_request=prompt_adapter_request,
429
+ )
430
327
- return self.input_processor(llm_inputs)
431
+ return self.input_processor(model_inputs)
432
433
async def add_request_async(
434
self,
@@ -336,6 +440,7 @@ class _AsyncLLMEngine(LLMEngine):
440
trace_headers: Optional[Mapping[str, str]] = None,
441
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
442
) -> None:
443
+ """Async version of :meth:`add_request`."""
444
if lora_request is not None and not self.lora_config:
445
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
446
"not enabled!")
@@ -343,10 +448,11 @@ class _AsyncLLMEngine(LLMEngine):
448
arrival_time = time.time()
449
450
processed_inputs = await self.process_model_inputs_async(
451
+ inputs,
452
request_id=request_id,
347
- inputs=inputs,
453
lora_request=lora_request,
349
- prompt_adapter_request=prompt_adapter_request)
454
+ prompt_adapter_request=prompt_adapter_request,
455
+ )
456
457
self._add_processed_request(
458
request_id=request_id,
vllm/engine/llm_engine.py
+151
-171
@@ -5,6 +5,8 @@ from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5
from typing import Sequence as GenericSequence
6
from typing import Set, Tuple, Type, TypeVar, Union
7
8
+from typing_extensions import assert_never
9
+
10
import vllm.envs as envs
11
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -22,10 +24,12 @@ from vllm.engine.output_processor.stop_checker import StopChecker
24
from vllm.engine.output_processor.util import create_output_by_sequence_group
25
from vllm.executor.executor_base import ExecutorBase
26
from vllm.executor.ray_utils import initialize_ray_cluster
25
-from vllm.inputs import (INPUT_REGISTRY, LLMInputs, PromptInputs,
26
- get_prompt_type)
27
+from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs, LLMInputs,
28
+ PromptInputs, SingletonPromptInputs)
29
+from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
30
from vllm.logger import init_logger
31
from vllm.lora.request import LoRARequest
32
+from vllm.multimodal import MultiModalDataDict
33
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
34
RequestOutputFactory)
35
from vllm.pooling_params import PoolingParams
@@ -43,8 +47,7 @@ from vllm.transformers_utils.tokenizer_group import (
47
AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
48
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
49
usage_message)
46
-from vllm.utils import (Counter, is_embedding_model_config,
47
- is_encoder_decoder_model_config)
50
+from vllm.utils import Counter
51
from vllm.version import __version__ as VLLM_VERSION
52
53
logger = init_logger(__name__)
@@ -66,6 +69,11 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
69
70
_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
71
72
+PromptComponents = Tuple[Optional[str], List[int],
73
+ Optional[MultiModalDataDict]]
74
+DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
75
+ Optional[MultiModalDataDict]]
76
+
77
78
class LLMEngine:
79
"""An LLM engine that receives requests and generates texts.
@@ -524,7 +532,7 @@ class LLMEngine:
532
533
return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
534
527
- def _get_decoder_start_token_id(self, ) -> Optional[int]:
535
+ def _get_decoder_start_token_id(self) -> Optional[int]:
536
'''
537
Obtain the decoder start token id employed by an encoder/decoder
538
model. Returns None for non-encoder/decoder models or if the
@@ -553,7 +561,7 @@ class LLMEngine:
561
def _add_processed_request(
562
self,
563
request_id: str,
556
- processed_inputs: LLMInputs,
564
+ processed_inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
565
params: Union[SamplingParams, PoolingParams],
566
arrival_time: float,
567
lora_request: Optional[LoRARequest],
@@ -613,11 +621,11 @@ class LLMEngine:
621
def stop_remote_worker_execution_loop(self) -> None:
622
self.model_executor.stop_remote_worker_execution_loop()
623
616
- _LLMInputComponentsType = Tuple[str, List[int], ]
624
+ _LLMInputComponentsType = Tuple[str, List[int]]
625
626
def _prepare_decoder_input_ids_for_generation(
627
self,
620
- decoder_input_ids: Optional[List[int]] = None,
628
+ decoder_input_ids: Optional[List[int]],
629
) -> List[int]:
630
"""
631
Prepares `decoder_input_ids` for generation with encoder-decoder models.
@@ -639,14 +647,13 @@ class LLMEngine:
647
* Processed token list
648
"""
649
642
- decoder_start_token_id: Optional[int] = (
643
- self._get_decoder_start_token_id())
650
+ decoder_start_token_id = self._get_decoder_start_token_id()
651
assert decoder_start_token_id is not None
652
653
if decoder_input_ids is None:
654
# no decoder prompt input ->
655
# use decoder_start_token_id as decoder_input_ids
649
- (decoder_input_ids) = self._get_default_enc_dec_decoder_prompt()
656
+ decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
657
658
if (len(decoder_input_ids) == 0
659
or decoder_input_ids[0] != decoder_start_token_id):
@@ -657,12 +664,11 @@ class LLMEngine:
664
def _tokenize_prompt(
665
self,
666
prompt: str,
660
- request_id: Optional[str] = None,
661
- lora_request: Optional[str] = None,
667
+ request_id: str,
668
+ lora_request: Optional[LoRARequest],
669
) -> List[int]:
670
'''
664
- Wrapper around application of the model's
665
- tokenizer.
671
+ Wrapper around application of the model's tokenizer.
672
673
Arguments:
674
@@ -678,87 +684,72 @@ class LLMEngine:
684
tokenizer = self.get_tokenizer_group("prompts must be None if "
685
"skip_tokenizer_init is True")
686
681
- prompt_token_ids = tokenizer.encode(request_id=request_id,
682
- prompt=prompt,
683
- lora_request=lora_request)
684
-
685
- return prompt_token_ids
687
+ return tokenizer.encode(request_id=request_id,
688
+ prompt=prompt,
689
+ lora_request=lora_request)
690
687
- def _extract_single_prompt_for_enc_dec_input(
691
+ def _extract_prompt_components(
692
self,
689
- inputs: Optional[PromptInputs],
690
- request_id: Optional[str] = None,
691
- ptype: Optional[str] = None,
692
- is_encoder_prompt: bool = False,
693
- ) -> Tuple[Optional[str], List[int]]:
693
+ inputs: SingletonPromptInputs,
694
+ request_id: str,
695
+ lora_request: Optional[LoRARequest] = None,
696
+ ) -> PromptComponents:
697
'''
695
- Only for encoder/decoder models:
696
- Extract prompt & prompt_token_ids from any single
697
- encoder or decoder input prompt. For encoder input prompts
698
- in particular, also extract multi-modal data.
699
-
700
- This function handles the following scenarios:
701
- 1. The user supplied a singleton encoder prompt
702
- & the prompt/prompt-token-ids must be extracted.
703
- 2. The user supplied an explicit encoder/decoder
704
- prompt & the prompt/prompt-token-ids must be
705
- extracted from either the encoder and decoder prompts.
706
-
707
- For decoder prompts in particular (scenario 2), special
708
- processing is applied to the returned decoder token ids.
698
+ Extract the components of any single encoder or decoder input prompt.
699
700
Arguments:
701
702
* request_id
713
- * ptype: str representation of the input prompt type.
714
- If `ptype` is `None`, assume that the prompt
715
- type is unknown and must be inferred. This is the
716
- case for ExplicitEncoderDecoder sub-prompts.
703
* inputs: single encoder or decoder input prompt
718
- * is_encoder_prompt: True if encoder input prompt.
719
- If False, decoder prompt tokens
720
- are preprocessed.
704
+ * lora_request: this is only valid for decoder prompts
705
706
Returns:
707
708
* prompt
709
* prompt_token_ids
710
+ * multi_modal_data
711
'''
727
- prompt_token_ids = None
728
- ptype = (get_prompt_type(inputs) if ptype is None else ptype)
712
730
- if inputs is None:
731
- prompt = None
732
- elif ptype == 'str':
713
+ if isinstance(inputs, str):
714
prompt = inputs
715
prompt_token_ids = self._tokenize_prompt(
716
prompt,
717
request_id=request_id,
718
+ lora_request=lora_request,
719
)
738
- elif ptype == 'TokensPrompt':
739
- prompt = None
740
- prompt_token_ids = inputs['prompt_token_ids']
720
+ multi_modal_data = None
721
+ elif isinstance(inputs, dict):
722
+ if "prompt_token_ids" in inputs:
723
+ prompt = None
724
+ prompt_token_ids = inputs["prompt_token_ids"]
725
+ else:
726
+ # NOTE: This extra assignment is required to pass mypy
727
+ prompt = parsed_prompt = inputs["prompt"]
728
+ prompt_token_ids = self._tokenize_prompt(
729
+ parsed_prompt,
730
+ request_id=request_id,
731
+ lora_request=lora_request,
732
+ )
733
+
734
+ multi_modal_data = inputs.get("multi_modal_data")
735
else:
742
- prompt = inputs['prompt']
743
- prompt_token_ids = self._tokenize_prompt(
744
- prompt,
745
- request_id=request_id,
746
- )
736
+ assert_never(inputs)
737
748
- if not is_encoder_prompt:
749
- # Apply special pre-processing to
750
- # decoder prompts
751
- prompt_token_ids = (self._prepare_decoder_input_ids_for_generation(
752
- prompt_token_ids, ))
738
+ return prompt, prompt_token_ids, multi_modal_data
739
754
- assert prompt_token_ids is not None
740
+ def _apply_prompt_adapter(
741
+ self,
742
+ prompt_token_ids: List[int],
743
+ prompt_adapter_request: Optional[PromptAdapterRequest],
744
+ ) -> List[int]:
745
+ if prompt_adapter_request:
746
+ prompt_token_ids = (
747
+ [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
748
+ + prompt_token_ids)
749
756
- return (
757
- prompt,
758
- prompt_token_ids,
759
- )
750
+ return prompt_token_ids
751
761
- def _get_default_enc_dec_decoder_prompt(self, ) -> List[int]:
752
+ def _get_default_enc_dec_decoder_prompt(self) -> List[int]:
753
'''
754
Specifically for encoder/decoder models:
755
generate a default decoder prompt for when
@@ -792,18 +783,39 @@ class LLMEngine:
783
784
bos_token_id = self._get_bos_token_id()
785
assert bos_token_id is not None
795
- prompt_token_ids: List[int] = [bos_token_id]
796
- return prompt_token_ids
786
+ return [bos_token_id]
787
+
788
+ def _build_enc_dec_llm_inputs(
789
+ self,
790
+ encoder_comps: PromptComponents,
791
+ decoder_comps: DecoderPromptComponents,
792
+ ) -> EncoderDecoderLLMInputs:
793
+ encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
794
+ decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
795
+
796
+ if encoder_mm_data is not None or decoder_mm_data is not None:
797
+ raise ValueError("Multi-modal encoder-decoder models are "
798
+ "not supported yet")
799
+
800
+ decoder_prompt_ids = (
801
+ self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
802
+
803
+ return EncoderDecoderLLMInputs(
804
+ prompt_token_ids=decoder_prompt_ids,
805
+ prompt=decoder_prompt,
806
+ encoder_prompt_token_ids=encoder_prompt_ids,
807
+ encoder_prompt=encoder_prompt,
808
+ )
809
810
def _process_encoder_decoder_prompt(
811
self,
812
inputs: PromptInputs,
801
- request_id: Optional[str] = None,
802
- ) -> LLMInputs:
813
+ request_id: str,
814
+ ) -> EncoderDecoderLLMInputs:
815
'''
816
For encoder/decoder models only:
805
- Process an input prompt
806
- into an `LLMInputs` instance.
817
+ Process an input prompt into an
818
+ :class:`EncoderDecoderLLMInputs` instance.
819
820
There are two types of input prompts:
821
singleton prompts which carry only the
@@ -830,136 +842,103 @@ class LLMEngine:
842
843
Returns:
844
833
- * `LLMInputs` instance
845
+ * :class:`EncoderDecoderLLMInputs` instance
846
'''
847
836
- ptype = get_prompt_type(inputs)
837
-
838
- # Obtain encoder and decoder prompt tokens. Note
839
- # that, no matter what, the decoder
840
- # prompt type is unknown.
841
- if ptype == "ExplicitEncoderDecoder":
842
- # If input is explicit encoder/decoder prompt,
843
- # then it remains to be determined what type
844
- # of encoder prompt we have
845
- extracted_encoder_prompt = inputs.get('encoder_prompt')
846
- encoder_ptype = None
847
- # Extract decoder prompt from explicit
848
- # encoder/decoder prompt
849
- extracted_decoder_prompt = inputs.get('decoder_prompt')
848
+ encoder_comps: PromptComponents
849
+ decoder_comps: DecoderPromptComponents
850
+
851
+ if is_explicit_encoder_decoder_prompt(inputs):
852
+ encoder_comps = self._extract_prompt_components(
853
+ inputs["encoder_prompt"],
854
+ request_id=request_id,
855
+ )
856
+
857
+ if (decoder_input := inputs["decoder_prompt"]) is None:
858
+ decoder_comps = None, None, None
859
+ else:
860
+ decoder_comps = self._extract_prompt_components(
861
+ decoder_input,
862
+ request_id=request_id,
863
+ )
864
else:
851
- # If input is singleton encoder prompt, then
852
- # we know the encoder prompt type
853
- extracted_encoder_prompt = inputs
854
- encoder_ptype = ptype
855
- # Decoder prompt is always unknown if
856
- # encoder/decoder prompt is not explicit
857
- extracted_decoder_prompt = None
858
-
859
- # Invoke helper function to obtain encoder
860
- # prompt and prompt token ids, either from
861
- # singleton encoder prompt or from the
862
- # encoder sub-prompt of an explicit
863
- # encoder/decode scenario 2), special
864
- # processing is applied to the returned decoder token ids
865
- (
866
- encoder_prompt,
867
- encoder_prompt_token_ids,
868
- ) = self._extract_single_prompt_for_enc_dec_input(
869
- extracted_encoder_prompt,
870
- request_id=request_id,
871
- ptype=encoder_ptype,
872
- is_encoder_prompt=True,
873
- )
865
+ encoder_comps = self._extract_prompt_components(
866
+ inputs,
867
+ request_id=request_id,
868
+ )
869
875
- # Invoke helper method to obtain
876
- # decoder prompt and prompt token ids.
877
- #
878
- # The helper method will detect the decoder
879
- # prompt type.
880
- #
881
- # Helper method will also apply special
882
- # preprocessing unique to decoder prompts.
883
- (
884
- decoder_prompt,
885
- decoder_prompt_token_ids,
886
- ) = self._extract_single_prompt_for_enc_dec_input(
887
- extracted_decoder_prompt,
888
- request_id=request_id,
889
- ptype=None,
890
- is_encoder_prompt=False,
891
- )
870
+ decoder_comps = None, None, None
871
893
- return LLMInputs(
894
- prompt_token_ids=decoder_prompt_token_ids,
895
- prompt=decoder_prompt,
896
- encoder_prompt_token_ids=encoder_prompt_token_ids,
897
- encoder_prompt=encoder_prompt,
898
- )
872
+ return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
873
+
874
+ def _build_decoder_only_llm_inputs(
875
+ self,
876
+ prompt_comps: PromptComponents,
877
+ prompt_adapter_request: Optional[PromptAdapterRequest],
878
+ ) -> LLMInputs:
879
+ prompt, prompt_token_ids, multi_modal_data = prompt_comps
880
+
881
+ prompt_token_ids = self._apply_prompt_adapter(
882
+ prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
883
+
884
+ return LLMInputs(prompt_token_ids=prompt_token_ids,
885
+ prompt=prompt,
886
+ multi_modal_data=multi_modal_data)
887
888
def _process_decoder_only_prompt(
889
self,
902
- inputs: PromptInputs,
890
+ inputs: SingletonPromptInputs,
891
+ request_id: str,
892
lora_request: Optional[LoRARequest] = None,
904
- request_id: Optional[str] = None,
893
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
894
) -> LLMInputs:
895
'''
896
For decoder-only models:
909
- Process an input prompt
910
- into an `LLMInputs` instance.
897
+ Process an input prompt into an :class:`LLMInputs` instance.
898
899
Arguments:
900
901
* inputs: input prompt
915
- * lora_request
902
* request_id
903
+ * lora_request
904
* prompt_adapter_request
905
906
Returns:
907
921
- * `LLMInputs` instance
908
+ * :class:`LLMInputs` instance
909
'''
910
924
- if isinstance(inputs, str):
925
- inputs = {"prompt": inputs}
926
- prompt = inputs.get("prompt")
927
-
928
- if "prompt_token_ids" not in inputs:
929
- prompt_token_ids = self._tokenize_prompt(
930
- prompt,
931
- request_id=request_id,
932
- lora_request=lora_request,
933
- )
934
- else:
935
- prompt_token_ids = inputs["prompt_token_ids"]
936
-
937
- if prompt_adapter_request:
938
- prompt_token_ids = (
939
- [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
940
- + prompt_token_ids)
911
+ prompt_comps = self._extract_prompt_components(
912
+ inputs,
913
+ request_id=request_id,
914
+ lora_request=lora_request,
915
+ )
916
942
- return LLMInputs(prompt_token_ids=prompt_token_ids,
943
- prompt=prompt,
944
- multi_modal_data=inputs.get("multi_modal_data"))
917
+ return self._build_decoder_only_llm_inputs(
918
+ prompt_comps,
919
+ prompt_adapter_request=prompt_adapter_request,
920
+ )
921
922
def process_model_inputs(
923
self,
948
- request_id: str,
924
inputs: PromptInputs,
925
+ request_id: str,
926
lora_request: Optional[LoRARequest] = None,
927
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
952
- ) -> LLMInputs:
928
+ ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
929
930
if self.is_encoder_decoder_model():
931
# Encoder-decoder model requires special mapping of
932
# input prompts to encoder & decoder
957
-
933
model_inputs = self._process_encoder_decoder_prompt(
934
inputs,
935
request_id=request_id,
936
)
937
else:
938
+ if is_explicit_encoder_decoder_prompt(inputs):
939
+ raise ValueError("Cannot pass encoder-decoder prompt "
940
+ "to decoder-only models")
941
+
942
# Decoder-only operation
943
model_inputs = self._process_decoder_only_prompt(
944
inputs,
@@ -1029,10 +1008,11 @@ class LLMEngine:
1008
arrival_time = time.time()
1009
1010
processed_inputs = self.process_model_inputs(
1011
+ inputs,
1012
request_id=request_id,
1033
- inputs=inputs,
1013
lora_request=lora_request,
1035
- prompt_adapter_request=prompt_adapter_request)
1014
+ prompt_adapter_request=prompt_adapter_request,
1015
+ )
1016
1017
self._add_processed_request(
1018
request_id=request_id,
@@ -1597,7 +1577,7 @@ class LLMEngine:
1577
seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
1578
1579
def is_encoder_decoder_model(self):
1600
- return is_encoder_decoder_model_config(self.model_config)
1580
+ return self.model_config.is_encoder_decoder_model
1581
1582
def is_embedding_model(self):
1603
- return is_embedding_model_config(self.model_config)
1583
+ return self.model_config.is_embedding_model
vllm/entrypoints/chat_utils.py
+2
-3
@@ -2,8 +2,7 @@ import codecs
2
from dataclasses import dataclass
3
from functools import lru_cache
4
from pathlib import Path
5
-from typing import (Any, Awaitable, Iterable, List, Optional, Tuple, Union,
6
- cast, final)
5
+from typing import Any, Awaitable, Iterable, List, Optional, Tuple, Union, cast
6
7
# yapf conflicts with isort for this block
8
# yapf: disable
@@ -59,7 +58,7 @@ ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
58
CustomChatCompletionMessageParam]
59
60
62
-@final # So that it should be compatible with Dict[str, str]
61
+# TODO: Make fields ReadOnly once mypy supports it
62
class ConversationMessage(TypedDict):
63
role: str
64
content: str
vllm/entrypoints/llm.py
+2
-2
@@ -6,8 +6,8 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.llm_engine import LLMEngine
9
-from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt,
10
- parse_and_batch_prompt)
9
+from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
10
+from vllm.inputs.parse import parse_and_batch_prompt
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
from vllm.model_executor.guided_decoding import (
vllm/entrypoints/openai/logits_processors.py
+5
-3
@@ -40,9 +40,11 @@ def _get_allowed_token_ids_logits_processor(
40
return AllowedTokenIdsLogitsProcessor(allowed_token_ids)
41
42
43
-def logit_bias_logits_processor(logit_bias: Dict[str,
44
- float], token_ids: List[int],
45
- logits: torch.Tensor) -> torch.Tensor:
43
+def logit_bias_logits_processor(
44
+ logit_bias: Dict[int, float],
45
+ token_ids: List[int],
46
+ logits: torch.Tensor,
47
+) -> torch.Tensor:
48
for token_id, bias in logit_bias.items():
49
logits[token_id] += bias
50
return logits
vllm/entrypoints/openai/serving_engine.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
22
TokenizeCompletionRequest,
23
TokenizeRequest)
24
# yapf: enable
25
-from vllm.inputs import parse_and_batch_prompt
25
+from vllm.inputs.parse import parse_and_batch_prompt
26
from vllm.logger import init_logger
27
from vllm.lora.request import LoRARequest
28
from vllm.model_executor.guided_decoding import (
vllm/inputs/__init__.py
+10
-11
@@ -1,7 +1,7 @@
1
-from .data import (ExplicitEncoderDecoderPrompt, LLMInputs, ParsedText,
2
- ParsedTokens, PromptInputs, SingletonPromptInputs,
3
- TextPrompt, TokensPrompt, get_prompt_type,
4
- is_valid_encoder_decoder_llm_inputs, parse_and_batch_prompt)
1
+from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2
+ LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
3
+ TokensPrompt, build_explicit_enc_dec_prompt,
4
+ to_enc_dec_tuple_list, zip_enc_dec_prompts)
5
from .registry import InputContext, InputRegistry
6
7
INPUT_REGISTRY = InputRegistry()
@@ -14,18 +14,17 @@ See also:
14
"""
15
16
__all__ = [
17
- "ParsedText",
18
- "ParsedTokens",
19
- "parse_and_batch_prompt",
17
"TextPrompt",
18
"TokensPrompt",
19
"PromptInputs",
20
+ "SingletonPromptInputs",
21
+ "ExplicitEncoderDecoderPrompt",
22
"LLMInputs",
23
+ "EncoderDecoderLLMInputs",
24
+ "build_explicit_enc_dec_prompt",
25
+ "to_enc_dec_tuple_list",
26
+ "zip_enc_dec_prompts",
27
"INPUT_REGISTRY",
28
"InputContext",
29
"InputRegistry",
27
- "get_prompt_type",
28
- "is_valid_encoder_decoder_llm_inputs",
29
- "ExplicitEncoderDecoderPrompt",
30
- "SingletonPromptInputs",
30
]
vllm/inputs/data.py
+72
-135
@@ -1,71 +1,12 @@
1
-from typing import (TYPE_CHECKING, List, Literal, Optional, Sequence,
2
- TypedDict, Union, cast, overload)
1
+from typing import (TYPE_CHECKING, Generic, Iterable, List, Optional, Tuple,
2
+ Union)
3
4
-from typing_extensions import NotRequired
4
+from typing_extensions import NotRequired, TypedDict, TypeVar
5
6
if TYPE_CHECKING:
7
from vllm.multimodal import MultiModalDataDict
8
9
10
-class ParsedText(TypedDict):
11
- content: str
12
- is_tokens: Literal[False]
13
-
14
-
15
-class ParsedTokens(TypedDict):
16
- content: List[int]
17
- is_tokens: Literal[True]
18
-
19
-
20
-# https://github.com/vllm-project/vllm/pull/4028
21
-@overload
22
-def parse_and_batch_prompt(
23
- prompt: Union[str, List[str]]) -> Sequence[ParsedText]:
24
- ...
25
-
26
-
27
-@overload
28
-def parse_and_batch_prompt(
29
- prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]:
30
- ...
31
-
32
-
33
-def parse_and_batch_prompt(
34
- prompt: Union[str, List[str], List[int], List[List[int]]],
35
-) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
36
- if isinstance(prompt, str):
37
- # case 1: a string
38
- return [ParsedText(content=prompt, is_tokens=False)]
39
-
40
- if isinstance(prompt, list):
41
- if len(prompt) == 0:
42
- raise ValueError("please provide at least one prompt")
43
-
44
- if isinstance(prompt[0], str):
45
- # case 2: array of strings
46
- return [
47
- ParsedText(content=elem, is_tokens=False)
48
- for elem in cast(List[str], prompt)
49
- ]
50
- if isinstance(prompt[0], int):
51
- # case 3: array of tokens
52
- elem = cast(List[int], prompt)
53
- return [ParsedTokens(content=elem, is_tokens=True)]
54
- if isinstance(prompt[0], list):
55
- if len(prompt[0]) == 0:
56
- raise ValueError("please provide at least one prompt")
57
-
58
- if isinstance(prompt[0][0], int):
59
- # case 4: array of token arrays
60
- return [
61
- ParsedTokens(content=elem, is_tokens=True)
62
- for elem in cast(List[List[int]], prompt)
63
- ]
64
-
65
- raise ValueError("prompt must be a string, array of strings, "
66
- "array of tokens, or array of token arrays")
67
-
68
-
10
class TextPrompt(TypedDict):
11
"""Schema for a text prompt."""
12
@@ -103,39 +44,49 @@ Note that "singleton" is as opposed to a data structure
44
which encapsulates multiple prompts, i.e. of the sort
45
which may be utilized for encoder/decoder models when
46
the user desires to express both the encoder & decoder
106
-prompts explicitly, i.e. ExplicitEncoderDecoderPrompt
47
+prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
108
-A prompt of type SingletonPromptInputs may be employed
49
+A prompt of type :class:`SingletonPromptInputs` may be employed
50
as (1) input to a decoder-only model, (2) input to
51
the encoder of an encoder/decoder model, in the scenario
52
where the decoder-prompt is not specified explicitly, or
53
(3) as a member of a larger data structure encapsulating
113
-more than one prompt, i.e. ExplicitEncoderDecoderPrompt
54
+more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55
"""
56
57
+_T1_co = TypeVar("_T1_co",
58
+ bound=SingletonPromptInputs,
59
+ default=SingletonPromptInputs,
60
+ covariant=True)
61
+_T2_co = TypeVar("_T2_co",
62
+ bound=SingletonPromptInputs,
63
+ default=SingletonPromptInputs,
64
+ covariant=True)
65
117
-class ExplicitEncoderDecoderPrompt(TypedDict):
66
+
67
+# TODO: Make fields ReadOnly once mypy supports it
68
+class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
69
"""Represents an encoder/decoder model input prompt,
70
comprising an explicit encoder prompt and a
71
decoder prompt.
72
73
The encoder and decoder prompts, respectively,
74
may formatted according to any of the
124
- SingletonPromptInputs schemas, and are not
75
+ :class:`SingletonPromptInputs` schemas, and are not
76
required to have the same schema.
77
78
Only the encoder prompt may have multi-modal data.
79
129
- Note that an ExplicitEncoderDecoderPrompt may not
80
+ Note that an :class:`ExplicitEncoderDecoderPrompt` may not
81
be used as an input to a decoder-only model,
82
and that the `encoder_prompt` and `decoder_prompt`
132
- fields of this data structure may not themselves
133
- must be SingletonPromptInputs instances.
83
+ fields of this data structure themselves must be
84
+ :class:`SingletonPromptInputs` instances.
85
"""
86
136
- encoder_prompt: SingletonPromptInputs
87
+ encoder_prompt: _T1_co
88
138
- decoder_prompt: SingletonPromptInputs
89
+ decoder_prompt: Optional[_T2_co]
90
91
92
PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
@@ -150,60 +101,12 @@ both decoder-only and encoder/decoder input types:
101
"""
102
103
153
-def _has_required_keys(
154
- d: dict,
155
- required_keys: set,
156
-) -> bool:
157
- return required_keys.issubset(d.keys())
158
-
159
-
160
-def get_prompt_type(prompt: Optional[PromptInputs]) -> Optional[str]:
161
- """
162
- Get the type-name of the prompt argument instance, given that
163
- isinstance() cannot apply to TypedDict subclasses directly.
164
- If the prompt is None, return 'None' as the type name.
165
-
166
- Arguments:
167
-
168
- * prompt: LLM input prompt or None
169
-
170
- Returns:
171
-
172
- * String representation of prompt type
173
- """
174
-
175
- if prompt is None:
176
- return 'None'
177
-
178
- required_keys_dict = {
179
- 'TextPrompt': {'prompt'},
180
- 'TokensPrompt': {'prompt_token_ids'},
181
- 'ExplicitEncoderDecoder': {'encoder_prompt', 'decoder_prompt'},
182
- }
183
-
184
- if isinstance(prompt, dict):
185
- for (ptype, required_keys) in required_keys_dict.items():
186
- # Ignore type checking in the conditional below because type
187
- # checker does not understand that is_dict(prompt) narrows
188
- # down the possible types
189
- if _has_required_keys(
190
- prompt, # type: ignore
191
- required_keys):
192
- return ptype
193
-
194
- raise ValueError(f"Invalid prompt {prompt}, valid types are "
195
- "required_keys_dict={required_keys_dict}")
196
-
197
- if isinstance(prompt, str):
198
- return "str"
199
-
200
- raise ValueError(f"Invalid prompt {prompt}")
201
-
202
-
104
class LLMInputs(TypedDict):
105
"""
106
The inputs in :class:`~vllm.LLMEngine` before they are
107
passed to the model executor.
108
+
109
+ This specifies the data required for decoder-only models.
110
"""
111
prompt_token_ids: List[int]
112
"""The token IDs of the prompt."""
@@ -213,7 +116,21 @@ class LLMInputs(TypedDict):
116
The original prompt text corresponding to the token IDs, if available.
117
"""
118
216
- encoder_prompt_token_ids: NotRequired[List[int]]
119
+ multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
120
+ """
121
+ Optional multi-modal data to pass to the model,
122
+ if the model supports it.
123
+ """
124
+
125
+
126
+class EncoderDecoderLLMInputs(LLMInputs):
127
+ """
128
+ The inputs in :class:`~vllm.LLMEngine` before they are
129
+ passed to the model executor.
130
+
131
+ This specifies the required data for encoder-decoder models.
132
+ """
133
+ encoder_prompt_token_ids: List[int]
134
"""The token IDs of the encoder prompt."""
135
136
encoder_prompt: NotRequired[Optional[str]]
@@ -222,20 +139,40 @@ class LLMInputs(TypedDict):
139
available.
140
"""
141
225
- multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
226
- """
227
- Optional multi-modal data to pass to the model,
228
- if the model supports it.
229
- """
142
+
143
+_T1 = TypeVar("_T1",
144
+ bound=SingletonPromptInputs,
145
+ default=SingletonPromptInputs)
146
+_T2 = TypeVar("_T2",
147
+ bound=SingletonPromptInputs,
148
+ default=SingletonPromptInputs)
149
150
232
-def is_valid_encoder_decoder_llm_inputs(inputs: LLMInputs) -> bool:
151
+def build_explicit_enc_dec_prompt(
152
+ encoder_prompt: _T1,
153
+ decoder_prompt: Optional[_T2],
154
+) -> ExplicitEncoderDecoderPrompt[_T1, _T2]:
155
+ return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
156
+ decoder_prompt=decoder_prompt)
157
+
158
+
159
+def zip_enc_dec_prompts(
160
+ enc_prompts: Iterable[_T1],
161
+ dec_prompts: Iterable[Optional[_T2]],
162
+) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
163
"""
234
- Return True if the LLMInputs instance has the correct configuration
235
- for encoder/decoder.
164
+ Zip encoder and decoder prompts together into a list of
165
+ :class:`ExplicitEncoderDecoderPrompt` instances.
166
"""
167
+ return [
168
+ build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
169
+ for (encoder_prompt, decoder_prompt) in zip(enc_prompts, dec_prompts)
170
+ ]
171
+
172
238
- # True if encoder prompt token ids field exists &
239
- # is not None
240
- return ('encoder_prompt_token_ids' in inputs
241
- and inputs['encoder_prompt_token_ids'] is not None)
173
+def to_enc_dec_tuple_list(
174
+ enc_dec_prompts: Iterable[ExplicitEncoderDecoderPrompt[_T1, _T2]],
175
+) -> List[Tuple[_T1, Optional[_T2]]]:
176
+ return [(enc_dec_prompt["encoder_prompt"],
177
+ enc_dec_prompt["decoder_prompt"])
178
+ for enc_dec_prompt in enc_dec_prompts]
vllm/inputs/parse.py
new
+75
@@ -0,0 +1,75 @@
1
+from typing import List, Literal, Sequence, TypedDict, Union, overload
2
+
3
+from typing_extensions import TypeIs
4
+
5
+from vllm.utils import is_list_of
6
+
7
+from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8
+ LLMInputs, PromptInputs)
9
+
10
+
11
+class ParsedText(TypedDict):
12
+ content: str
13
+ is_tokens: Literal[False]
14
+
15
+
16
+class ParsedTokens(TypedDict):
17
+ content: List[int]
18
+ is_tokens: Literal[True]
19
+
20
+
21
+@overload
22
+def parse_and_batch_prompt(
23
+ prompt: Union[str, List[str]]) -> Sequence[ParsedText]:
24
+ ...
25
+
26
+
27
+@overload
28
+def parse_and_batch_prompt(
29
+ prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]:
30
+ ...
31
+
32
+
33
+def parse_and_batch_prompt(
34
+ prompt: Union[str, List[str], List[int], List[List[int]]],
35
+) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
36
+ if isinstance(prompt, str):
37
+ # case 1: a string
38
+ return [ParsedText(content=prompt, is_tokens=False)]
39
+
40
+ if isinstance(prompt, list):
41
+ if len(prompt) == 0:
42
+ raise ValueError("please provide at least one prompt")
43
+
44
+ if is_list_of(prompt, str):
45
+ # case 2: array of strings
46
+ return [
47
+ ParsedText(content=elem, is_tokens=False) for elem in prompt
48
+ ]
49
+ if is_list_of(prompt, int):
50
+ # case 3: array of tokens
51
+ return [ParsedTokens(content=prompt, is_tokens=True)]
52
+ if is_list_of(prompt, list):
53
+ if len(prompt[0]) == 0:
54
+ raise ValueError("please provide at least one prompt")
55
+
56
+ if is_list_of(prompt[0], int):
57
+ # case 4: array of token arrays
58
+ return [
59
+ ParsedTokens(content=elem, is_tokens=True)
60
+ for elem in prompt
61
+ ]
62
+
63
+ raise ValueError("prompt must be a string, array of strings, "
64
+ "array of tokens, or array of token arrays")
65
+
66
+
67
+def is_explicit_encoder_decoder_prompt(
68
+ inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
69
+ return isinstance(inputs, dict) and "encoder_prompt" in inputs
70
+
71
+
72
+def is_valid_encoder_decoder_llm_inputs(
73
+ inputs: Union[LLMInputs, EncoderDecoderLLMInputs],
74
+) -> TypeIs[EncoderDecoderLLMInputs]:
75
+ return "encoder_prompt_token_ids" in inputs
vllm/model_executor/models/interfaces.py
+11
-11
@@ -1,7 +1,7 @@
1
from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
2
Union, overload, runtime_checkable)
3
4
-from typing_extensions import TypeGuard
4
+from typing_extensions import TypeIs
5
6
from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
7
from vllm.logger import init_logger
@@ -37,18 +37,18 @@ class _SupportsVisionType(Protocol):
37
38
39
@overload
40
-def supports_vision(model: Type[object]) -> TypeGuard[Type[SupportsVision]]:
40
+def supports_vision(model: Type[object]) -> TypeIs[Type[SupportsVision]]:
41
...
42
43
44
@overload
45
-def supports_vision(model: object) -> TypeGuard[SupportsVision]:
45
+def supports_vision(model: object) -> TypeIs[SupportsVision]:
46
...
47
48
49
def supports_vision(
50
model: Union[Type[object], object],
51
-) -> Union[TypeGuard[Type[SupportsVision]], TypeGuard[SupportsVision]]:
51
+) -> Union[TypeIs[Type[SupportsVision]], TypeIs[SupportsVision]]:
52
if isinstance(model, type):
53
return isinstance(model, _SupportsVisionType)
54
@@ -94,18 +94,18 @@ class _SupportsLoRAType(Protocol):
94
95
96
@overload
97
-def supports_lora(model: Type[object]) -> TypeGuard[Type[SupportsLoRA]]:
97
+def supports_lora(model: Type[object]) -> TypeIs[Type[SupportsLoRA]]:
98
...
99
100
101
@overload
102
-def supports_lora(model: object) -> TypeGuard[SupportsLoRA]:
102
+def supports_lora(model: object) -> TypeIs[SupportsLoRA]:
103
...
104
105
106
def supports_lora(
107
model: Union[Type[object], object],
108
-) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
108
+) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
109
result = _supports_lora(model)
110
111
if not result:
@@ -137,7 +137,7 @@ def supports_lora(
137
138
def _supports_lora(
139
model: Union[Type[object], object],
140
-) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
140
+) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
141
if isinstance(model, type):
142
return isinstance(model, _SupportsLoRAType)
143
@@ -172,18 +172,18 @@ class _HasInnerStateType(Protocol):
172
173
174
@overload
175
-def has_inner_state(model: object) -> TypeGuard[HasInnerState]:
175
+def has_inner_state(model: object) -> TypeIs[HasInnerState]:
176
...
177
178
179
@overload
180
-def has_inner_state(model: Type[object]) -> TypeGuard[Type[HasInnerState]]:
180
+def has_inner_state(model: Type[object]) -> TypeIs[Type[HasInnerState]]:
181
...
182
183
184
def has_inner_state(
185
model: Union[Type[object], object]
186
-) -> Union[TypeGuard[Type[HasInnerState]], TypeGuard[HasInnerState]]:
186
+) -> Union[TypeIs[Type[HasInnerState]], TypeIs[HasInnerState]]:
187
if isinstance(model, type):
188
return isinstance(model, _HasInnerStateType)
189
vllm/multimodal/image.py
+4
-2
@@ -10,6 +10,7 @@ from vllm.inputs.registry import InputContext
10
from vllm.logger import init_logger
11
from vllm.transformers_utils.image_processor import get_image_processor
12
from vllm.transformers_utils.tokenizer import get_tokenizer
13
+from vllm.utils import is_list_of
14
15
from .base import MultiModalInputs, MultiModalPlugin
16
@@ -113,7 +114,8 @@ class ImagePlugin(MultiModalPlugin):
114
def _default_input_mapper(self, ctx: InputContext,
115
data: object) -> MultiModalInputs:
116
model_config = ctx.model_config
116
- if isinstance(data, (Image.Image, list)):
117
+
118
+ if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
119
image_processor = self._get_hf_image_processor(model_config)
120
if image_processor is None:
121
raise RuntimeError("No HuggingFace processor is available "
@@ -127,7 +129,7 @@ class ImagePlugin(MultiModalPlugin):
129
raise
130
131
return MultiModalInputs(batch_data)
130
- elif isinstance(data, torch.Tensor):
132
+ elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor):
133
raise NotImplementedError("Embeddings input is not supported yet")
134
135
raise TypeError(f"Invalid image type: {type(data)}")
vllm/sequence.py
+1
-1
@@ -11,7 +11,7 @@ from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
11
12
import torch
13
14
-from vllm.inputs import is_valid_encoder_decoder_llm_inputs
14
+from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
15
from vllm.lora.request import LoRARequest
16
from vllm.pooling_params import PoolingParams
17
from vllm.prompt_adapter.request import PromptAdapterRequest
vllm/utils.py
+22
-52
@@ -17,8 +17,8 @@ from collections import defaultdict
17
from functools import lru_cache, partial, wraps
18
from platform import uname
19
from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
20
- Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
21
- Union, overload)
20
+ Hashable, List, Literal, Optional, OrderedDict, Set, Tuple,
21
+ Type, TypeVar, Union, overload)
22
from uuid import uuid4
23
24
import numpy as np
@@ -26,12 +26,10 @@ import numpy.typing as npt
26
import psutil
27
import torch
28
import torch.types
29
-from typing_extensions import ParamSpec
29
+from typing_extensions import ParamSpec, TypeIs, assert_never
30
31
import vllm.envs as envs
32
from vllm import _custom_ops as ops
33
-from vllm.inputs import (ExplicitEncoderDecoderPrompt, PromptInputs,
34
- SingletonPromptInputs)
33
from vllm.logger import enable_trace_function_call, init_logger
34
35
logger = init_logger(__name__)
@@ -812,6 +810,24 @@ def get_dtype_size(dtype: torch.dtype) -> int:
810
return torch.tensor([], dtype=dtype).element_size()
811
812
813
+# `collections` helpers
814
+def is_list_of(
815
+ value: object,
816
+ typ: Type[T],
817
+ *,
818
+ check: Literal["first", "all"] = "first",
819
+) -> TypeIs[List[T]]:
820
+ if not isinstance(value, list):
821
+ return False
822
+
823
+ if check == "first":
824
+ return len(value) == 0 or isinstance(value[0], typ)
825
+ elif check == "all":
826
+ return all(isinstance(v, typ) for v in value)
827
+
828
+ assert_never(check)
829
+
830
+
831
def merge_dicts(dict1: Dict[K, List[T]],
832
dict2: Dict[K, List[T]]) -> Dict[K, List[T]]:
833
"""Merge 2 dicts that have key -> List of items.
@@ -959,6 +975,7 @@ def enable_trace_function_call_for_thread() -> None:
975
enable_trace_function_call(log_path)
976
977
978
+# `functools` helpers
979
def identity(value: T) -> T:
980
return value
981
@@ -1080,50 +1097,3 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1097
"""Utility function to run async task in a lock"""
1098
async with lock:
1099
return await task(*args, **kwargs)
1083
-
1084
-
1085
-def is_encoder_decoder_model_config(model_config) -> bool:
1086
- '''
1087
- Extract the HF encoder/decoder model flag from the ModelConfig instance.
1088
- Return False if model_config is None.
1089
- '''
1090
- return model_config is not None and \
1091
- getattr(model_config.hf_config,
1092
- "is_encoder_decoder",
1093
- False)
1094
-
1095
-
1096
-def is_embedding_model_config(model_config) -> bool:
1097
- '''
1098
- Extract the embedding model flag from the ModelConfig instance.
1099
- Return False if model_config is None.
1100
- '''
1101
- return model_config is not None and \
1102
- model_config.embedding_mode
1103
-
1104
-
1105
-def build_explicit_enc_dec_prompt(
1106
- encoder_prompt: SingletonPromptInputs,
1107
- decoder_prompt: SingletonPromptInputs,
1108
-) -> ExplicitEncoderDecoderPrompt:
1109
- return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
1110
- decoder_prompt=decoder_prompt)
1111
-
1112
-
1113
-def zip_enc_dec_prompt_lists(
1114
- enc_prompt_list: List[SingletonPromptInputs],
1115
- dec_prompt_list: List[SingletonPromptInputs],
1116
-) -> List[ExplicitEncoderDecoderPrompt]:
1117
- return [
1118
- build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
1119
- for (encoder_prompt,
1120
- decoder_prompt) in zip(enc_prompt_list, dec_prompt_list)
1121
- ]
1122
-
1123
-
1124
-def to_enc_dec_tuple_list(
1125
- enc_dec_prompts: List[ExplicitEncoderDecoderPrompt],
1126
-) -> List[Tuple[PromptInputs, PromptInputs]]:
1127
- return [(enc_dec_prompt['encoder_prompt'],
1128
- enc_dec_prompt['decoder_prompt'])
1129
- for enc_dec_prompt in enc_dec_prompts]
vllm/worker/worker.py
+2
-4
@@ -19,8 +19,6 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19
from vllm.platforms import current_platform
20
from vllm.prompt_adapter.request import PromptAdapterRequest
21
from vllm.sequence import ExecuteModelRequest
22
-from vllm.utils import (is_embedding_model_config,
23
- is_encoder_decoder_model_config)
22
from vllm.worker.cache_engine import CacheEngine
23
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
24
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
@@ -113,10 +111,10 @@ class Worker(LocalOrDistributedWorkerBase):
111
self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
112
113
def _is_encoder_decoder_model(self):
116
- return is_encoder_decoder_model_config(self.model_config)
114
+ return self.model_config.is_encoder_decoder_model
115
116
def _is_embedding_model(self):
119
- return is_embedding_model_config(self.model_config)
117
+ return self.model_config.is_embedding_model
118
119
def init_device(self) -> None:
120
if self.device_config.device.type == "cuda":