A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] Rename `PromptInputs` and `inputs`(#8673)
Cyrus Leung committed
Sep 21, 2024 at 10:00 UTC
0057894ef7f8db0d51385aa7254219d7fbd6c784
18 files changed
+157
-162
benchmarks/benchmark_latency.py
+4
-4
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12
from vllm import LLM, SamplingParams
13
from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14
-from vllm.inputs import PromptInputs
14
+from vllm.inputs import PromptType
15
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61
dummy_prompt_token_ids = np.random.randint(10000,
62
size=(args.batch_size,
63
args.input_len))
64
- dummy_inputs: List[PromptInputs] = [{
64
+ dummy_prompts: List[PromptType] = [{
65
"prompt_token_ids": batch
66
} for batch in dummy_prompt_token_ids.tolist()]
67
@@ -74,13 +74,13 @@ def main(args: argparse.Namespace):
74
],
75
on_trace_ready=torch.profiler.tensorboard_trace_handler(
76
str(profile_dir))) as p:
77
- llm.generate(dummy_inputs,
77
+ llm.generate(dummy_prompts,
78
sampling_params=sampling_params,
79
use_tqdm=False)
80
print(p.key_averages())
81
else:
82
start_time = time.perf_counter()
83
- llm.generate(dummy_inputs,
83
+ llm.generate(dummy_prompts,
84
sampling_params=sampling_params,
85
use_tqdm=False)
86
end_time = time.perf_counter()
docs/source/dev/multimodal/multimodal_index.rst
+1
-1
@@ -8,7 +8,7 @@ Multi-Modality
8
vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10
Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11
-via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
11
+via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
12
13
Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14
by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1
-1
@@ -1,7 +1,7 @@
1
LLM Inputs
2
==========
3
4
-.. autodata:: vllm.inputs.PromptInputs
4
+.. autodata:: vllm.inputs.PromptType
5
6
.. autoclass:: vllm.inputs.TextPrompt
7
:show-inheritance:
docs/source/models/vlm.rst
+1
-1
@@ -27,7 +27,7 @@ The :class:`~vllm.LLM` class can be instantiated in much the same way as languag
27
We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28
the above snippet. Specifically, ``image_feature_size`` can no longer be specified as we now calculate that internally for each model.
29
30
-To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
30
+To pass an image to the model, note the following in :class:`vllm.inputs.PromptType`:
31
32
* ``prompt``: The prompt should follow the format that is documented on HuggingFace.
33
* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/mq_llm_engine/test_error_handling.py
+6
-6
@@ -61,7 +61,7 @@ async def test_evil_forward(tmp_socket):
61
62
# Throws an error in first forward pass.
63
with pytest.raises(RAISED_ERROR):
64
- async for _ in client.generate(inputs="Hello my name is",
64
+ async for _ in client.generate(prompt="Hello my name is",
65
sampling_params=SamplingParams(),
66
request_id=uuid.uuid4()):
67
pass
@@ -69,7 +69,7 @@ async def test_evil_forward(tmp_socket):
69
70
# Engine is errored, should get ENGINE_DEAD_ERROR.
71
with pytest.raises(MQEngineDeadError):
72
- async for _ in client.generate(inputs="Hello my name is",
72
+ async for _ in client.generate(prompt="Hello my name is",
73
sampling_params=SamplingParams(),
74
request_id=uuid.uuid4()):
75
pass
@@ -118,7 +118,7 @@ async def test_failed_health_check(tmp_socket):
118
119
# Generate call should throw ENGINE_DEAD_ERROR
120
with pytest.raises(MQEngineDeadError):
121
- async for _ in client.generate(inputs="Hello my name is",
121
+ async for _ in client.generate(prompt="Hello my name is",
122
sampling_params=SamplingParams(),
123
request_id=uuid.uuid4()):
124
pass
@@ -165,7 +165,7 @@ async def test_failed_abort(tmp_socket):
165
# with reference to the original KeyError("foo")
166
with pytest.raises(MQEngineDeadError) as execinfo:
167
async for _ in client.generate(
168
- inputs="Hello my name is",
168
+ prompt="Hello my name is",
169
sampling_params=SamplingParams(max_tokens=2000),
170
request_id=uuid.uuid4()):
171
pass
@@ -190,7 +190,7 @@ async def test_bad_request(tmp_socket):
190
191
# Invalid request should fail, but not crash the server.
192
with pytest.raises(ValueError):
193
- async for _ in client.generate(inputs="Hello my name is",
193
+ async for _ in client.generate(prompt="Hello my name is",
194
sampling_params=SamplingParams(),
195
request_id="abcd-1",
196
lora_request=LoRARequest(
@@ -199,7 +199,7 @@ async def test_bad_request(tmp_socket):
199
pass
200
201
# This request should be okay.
202
- async for _ in client.generate(inputs="Hello my name is",
202
+ async for _ in client.generate(prompt="Hello my name is",
203
sampling_params=SamplingParams(),
204
request_id="abcd-2"):
205
pass
tests/mq_llm_engine/utils.py
+1
-1
@@ -20,7 +20,7 @@ async def generate(
20
count = 0
21
async for out in client.generate(
22
request_id=request_id,
23
- inputs="Hello my name is Robert and",
23
+ prompt="Hello my name is Robert and",
24
sampling_params=SamplingParams(max_tokens=num_tokens,
25
temperature=0)):
26
vllm/__init__.py
+2
-2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
-from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
8
+from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11
EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19
"__version__",
20
"LLM",
21
"ModelRegistry",
22
- "PromptInputs",
22
+ "PromptType",
23
"TextPrompt",
24
"TokensPrompt",
25
"SamplingParams",
vllm/engine/async_llm_engine.py
+11
-13
@@ -17,7 +17,7 @@ from vllm.engine.metrics_types import StatLoggerBase
17
from vllm.executor.executor_base import ExecutorAsyncBase
18
from vllm.executor.gpu_executor import GPUExecutorAsync
19
from vllm.executor.ray_utils import initialize_ray_cluster
20
-from vllm.inputs import PromptInputs
20
+from vllm.inputs import PromptType
21
from vllm.logger import init_logger
22
from vllm.lora.request import LoRARequest
23
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -405,7 +405,7 @@ class _AsyncLLMEngine(LLMEngine):
405
async def add_request_async(
406
self,
407
request_id: str,
408
- inputs: PromptInputs,
408
+ prompt: PromptType,
409
params: Union[SamplingParams, PoolingParams],
410
arrival_time: Optional[float] = None,
411
lora_request: Optional[LoRARequest] = None,
@@ -420,7 +420,7 @@ class _AsyncLLMEngine(LLMEngine):
420
arrival_time = time.time()
421
422
preprocessed_inputs = await self.input_preprocessor.preprocess_async(
423
- inputs,
423
+ prompt,
424
request_id=request_id,
425
lora_request=lora_request,
426
prompt_adapter_request=prompt_adapter_request,
@@ -777,7 +777,7 @@ class AsyncLLMEngine:
777
async def add_request(
778
self,
779
request_id: str,
780
- inputs: PromptInputs,
780
+ prompt: PromptType,
781
params: Union[SamplingParams, PoolingParams],
782
arrival_time: Optional[float] = None,
783
lora_request: Optional[LoRARequest] = None,
@@ -797,7 +797,7 @@ class AsyncLLMEngine:
797
stream = self._request_tracker.add_request(
798
request_id,
799
verbose=self.log_requests,
800
- inputs=inputs,
800
+ prompt=prompt,
801
params=params,
802
arrival_time=arrival_time or time.time(),
803
lora_request=lora_request,
@@ -808,7 +808,7 @@ class AsyncLLMEngine:
808
809
async def generate(
810
self,
811
- inputs: PromptInputs,
811
+ prompt: PromptType,
812
sampling_params: SamplingParams,
813
request_id: str,
814
lora_request: Optional[LoRARequest] = None,
@@ -822,8 +822,7 @@ class AsyncLLMEngine:
822
from the LLMEngine to the caller.
823
824
Args:
825
- inputs: The inputs to the LLM. See
826
- :class:`~vllm.inputs.PromptInputs`
825
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
826
for more details about the format of each input.
827
sampling_params: The sampling parameters of the request.
828
request_id: The unique id of the request.
@@ -881,7 +880,7 @@ class AsyncLLMEngine:
880
"""
881
async for output in await self.add_request(
882
request_id,
884
- inputs,
883
+ prompt,
884
sampling_params,
885
lora_request=lora_request,
886
trace_headers=trace_headers,
@@ -891,7 +890,7 @@ class AsyncLLMEngine:
890
891
async def encode(
892
self,
894
- inputs: PromptInputs,
893
+ prompt: PromptType,
894
pooling_params: PoolingParams,
895
request_id: str,
896
lora_request: Optional[LoRARequest] = None,
@@ -904,8 +903,7 @@ class AsyncLLMEngine:
903
from the LLMEngine to the caller.
904
905
Args:
907
- inputs: The inputs to the LLM. See
908
- :class:`~vllm.inputs.PromptInputs`
906
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
907
for more details about the format of each input.
908
pooling_params: The pooling parameters of the request.
909
request_id: The unique id of the request.
@@ -959,7 +957,7 @@ class AsyncLLMEngine:
957
"""
958
async for output in await self.add_request(
959
request_id,
962
- inputs,
960
+ prompt,
961
pooling_params,
962
lora_request=lora_request,
963
trace_headers=trace_headers,
vllm/engine/llm_engine.py
+4
-5
@@ -29,7 +29,7 @@ from vllm.executor.executor_base import ExecutorBase
29
from vllm.executor.gpu_executor import GPUExecutor
30
from vllm.executor.ray_utils import initialize_ray_cluster
31
from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32
- InputRegistry, LLMInputs, PromptInputs)
32
+ InputRegistry, LLMInputs, PromptType)
33
from vllm.inputs.preprocess import InputPreprocessor
34
from vllm.logger import init_logger
35
from vllm.lora.request import LoRARequest
@@ -680,7 +680,7 @@ class LLMEngine:
680
def add_request(
681
self,
682
request_id: str,
683
- inputs: PromptInputs,
683
+ prompt: PromptType,
684
params: Union[SamplingParams, PoolingParams],
685
arrival_time: Optional[float] = None,
686
lora_request: Optional[LoRARequest] = None,
@@ -695,8 +695,7 @@ class LLMEngine:
695
696
Args:
697
request_id: The unique ID of the request.
698
- inputs: The inputs to the LLM. See
699
- :class:`~vllm.inputs.PromptInputs`
698
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
699
for more details about the format of each input.
700
params: Parameters for sampling or pooling.
701
:class:`~vllm.SamplingParams` for text generation.
@@ -736,7 +735,7 @@ class LLMEngine:
735
arrival_time = time.time()
736
737
preprocessed_inputs = self.input_preprocessor.preprocess(
739
- inputs,
738
+ prompt,
739
request_id=request_id,
740
lora_request=lora_request,
741
prompt_adapter_request=prompt_adapter_request,
vllm/engine/multiprocessing/__init__.py
+2
-2
@@ -3,7 +3,7 @@ from enum import Enum
3
from typing import List, Mapping, Optional, Union
4
5
from vllm import PoolingParams
6
-from vllm.inputs import PromptInputs
6
+from vllm.inputs import PromptType
7
from vllm.lora.request import LoRARequest
8
from vllm.outputs import RequestOutput
9
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -23,7 +23,7 @@ class MQEngineDeadError(RuntimeError):
23
24
@dataclass
25
class RPCProcessRequest:
26
- inputs: PromptInputs
26
+ prompt: PromptType
27
params: Union[SamplingParams, PoolingParams]
28
request_id: str
29
lora_request: Optional[LoRARequest] = None
vllm/engine/multiprocessing/client.py
+9
-11
@@ -25,7 +25,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25
RPCStartupResponse)
26
# yapf: enable
27
from vllm.envs import VLLM_RPC_TIMEOUT
28
-from vllm.inputs import PromptInputs
28
+from vllm.inputs import PromptType
29
from vllm.logger import init_logger
30
from vllm.lora.request import LoRARequest
31
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -375,7 +375,7 @@ class MQLLMEngineClient:
375
376
def generate(
377
self,
378
- inputs: PromptInputs,
378
+ prompt: PromptType,
379
sampling_params: SamplingParams,
380
request_id: str,
381
lora_request: Optional[LoRARequest] = None,
@@ -389,8 +389,7 @@ class MQLLMEngineClient:
389
from the LLMEngine to the caller.
390
391
Args:
392
- inputs: The inputs to the LLM. See
393
- :class:`~vllm.inputs.PromptInputs`
392
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
393
for more details about the format of each input.
394
sampling_params: The sampling parameters of the request.
395
request_id: The unique id of the request.
@@ -399,13 +398,13 @@ class MQLLMEngineClient:
398
prompt_adapter_request: Prompt Adapter request to use
399
for generation, if any.
400
"""
402
- return self._process_request(inputs, sampling_params, request_id,
401
+ return self._process_request(prompt, sampling_params, request_id,
402
lora_request, trace_headers,
403
prompt_adapter_request)
404
405
def encode(
406
self,
408
- inputs: PromptInputs,
407
+ prompt: PromptType,
408
pooling_params: PoolingParams,
409
request_id: str,
410
lora_request: Optional[LoRARequest] = None,
@@ -418,8 +417,7 @@ class MQLLMEngineClient:
417
from the LLMEngine to the caller.
418
419
Args:
421
- inputs: The inputs to the LLM. See
422
- :class:`~vllm.inputs.PromptInputs`
420
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
421
for more details about the format of each input.
422
pooling_params: The pooling parameters of the request.
423
request_id: The unique id of the request.
@@ -430,12 +428,12 @@ class MQLLMEngineClient:
428
The output `EmbeddingRequestOutput` objects from the LLMEngine
429
for the request.
430
"""
433
- return self._process_request(inputs, pooling_params, request_id,
431
+ return self._process_request(prompt, pooling_params, request_id,
432
lora_request, trace_headers)
433
434
async def _process_request(
435
self,
438
- inputs: PromptInputs,
436
+ prompt: PromptType,
437
params: Union[SamplingParams, PoolingParams],
438
request_id: str,
439
lora_request: Optional[LoRARequest] = None,
@@ -468,7 +466,7 @@ class MQLLMEngineClient:
466
467
request_bytes = pickle.dumps(
468
RPCProcessRequest(
471
- inputs=inputs,
469
+ prompt=prompt,
470
params=params,
471
request_id=request_id,
472
lora_request=lora_request,
vllm/engine/multiprocessing/engine.py
+1
-1
@@ -245,7 +245,7 @@ class MQLLMEngine:
245
try:
246
self.engine.add_request(
247
request_id=request_id,
248
- inputs=request.inputs,
248
+ prompt=request.prompt,
249
params=request.params,
250
lora_request=request.lora_request,
251
trace_headers=request.trace_headers,
vllm/engine/protocol.py
+4
-4
@@ -3,7 +3,7 @@ from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
3
4
from vllm.config import DecodingConfig, ModelConfig
5
from vllm.core.scheduler import SchedulerOutputs
6
-from vllm.inputs.data import PromptInputs
6
+from vllm.inputs.data import PromptType
7
from vllm.lora.request import LoRARequest
8
from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -35,19 +35,19 @@ class EngineClient(Protocol):
35
36
def generate(
37
self,
38
- inputs: PromptInputs,
38
+ prompt: PromptType,
39
sampling_params: SamplingParams,
40
request_id: str,
41
lora_request: Optional[LoRARequest] = None,
42
trace_headers: Optional[Mapping[str, str]] = None,
43
prompt_adapter_request: Optional[PromptAdapterRequest] = None
44
) -> AsyncGenerator[RequestOutput, None]:
45
- """Generates outputs for a request"""
45
+ """Generate outputs for a request."""
46
...
47
48
def encode(
49
self,
50
- inputs: PromptInputs,
50
+ prompt: PromptType,
51
pooling_params: PoolingParams,
52
request_id: str,
53
lora_request: Optional[LoRARequest] = None,
vllm/entrypoints/llm.py
+42
-38
@@ -10,7 +10,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
10
apply_hf_chat_template,
11
apply_mistral_chat_template,
12
parse_chat_messages)
13
-from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
13
+from vllm.inputs import PromptType, TextPrompt, TokensPrompt
14
from vllm.inputs.parse import parse_and_batch_prompt
15
from vllm.logger import init_logger
16
from vllm.lora.request import LoRARequest
@@ -258,8 +258,8 @@ class LLM:
258
@overload
259
def generate(
260
self,
261
- inputs: Union[PromptInputs, Sequence[PromptInputs]],
262
- /, # We may enable `inputs` keyword after removing the old API
261
+ prompts: Union[PromptType, Sequence[PromptType]],
262
+ /,
263
*,
264
sampling_params: Optional[Union[SamplingParams,
265
Sequence[SamplingParams]]] = None,
@@ -276,7 +276,7 @@ class LLM:
276
)
277
def generate(
278
self,
279
- prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
279
+ prompts: Union[Union[PromptType, Sequence[PromptType]],
280
Optional[Union[str, List[str]]]] = None,
281
sampling_params: Optional[Union[SamplingParams,
282
Sequence[SamplingParams]]] = None,
@@ -294,7 +294,9 @@ class LLM:
294
into a single list and pass it to this method.
295
296
Args:
297
- inputs: A list of inputs to generate completions for.
297
+ prompts: The prompts to the LLM. You may pass a sequence of prompts
298
+ for batch inference. See :class:`~vllm.inputs.PromptType`
299
+ for more details about the format of each prompts.
300
sampling_params: The sampling parameters for text generation. If
301
None, we use the default sampling parameters.
302
When it is a single value, it is applied to every prompt.
@@ -320,12 +322,13 @@ class LLM:
322
"models (XForCausalLM, XForConditionalGeneration).")
323
324
if prompt_token_ids is not None:
323
- inputs = self._convert_v1_inputs(
325
+ parsed_prompts = self._convert_v1_inputs(
326
prompts=cast(Optional[Union[str, List[str]]], prompts),
327
prompt_token_ids=prompt_token_ids,
328
)
329
else:
328
- inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
330
+ parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
331
+ prompts)
332
333
if isinstance(guided_options_request, dict):
334
if len(guided_options_request) > 1:
@@ -340,7 +343,7 @@ class LLM:
343
sampling_params = SamplingParams()
344
345
self._validate_and_add_requests(
343
- inputs=inputs,
346
+ prompts=parsed_prompts,
347
params=sampling_params,
348
lora_request=lora_request,
349
prompt_adapter_request=prompt_adapter_request,
@@ -396,9 +399,9 @@ class LLM:
399
conversation, mm_data = parse_chat_messages(messages, model_config,
400
tokenizer)
401
399
- prompt: Union[str, List[int]]
402
+ prompt_data: Union[str, List[int]]
403
if isinstance(tokenizer, MistralTokenizer):
401
- prompt = apply_mistral_chat_template(
404
+ prompt_data = apply_mistral_chat_template(
405
tokenizer,
406
messages=messages,
407
chat_template=chat_template,
@@ -406,7 +409,7 @@ class LLM:
409
tools=tools,
410
)
411
else:
409
- prompt = apply_hf_chat_template(
412
+ prompt_data = apply_hf_chat_template(
413
tokenizer,
414
conversation=conversation,
415
chat_template=chat_template,
@@ -414,17 +417,17 @@ class LLM:
417
tools=tools,
418
)
419
417
- inputs: PromptInputs
418
- if is_list_of(prompt, int):
419
- inputs = TokensPrompt(prompt_token_ids=prompt)
420
+ prompt: PromptType
421
+ if is_list_of(prompt_data, int):
422
+ prompt = TokensPrompt(prompt_token_ids=prompt_data)
423
else:
421
- inputs = TextPrompt(prompt=prompt)
424
+ prompt = TextPrompt(prompt=prompt_data)
425
426
if mm_data is not None:
424
- inputs["multi_modal_data"] = mm_data
427
+ prompt["multi_modal_data"] = mm_data
428
429
return self.generate(
427
- inputs,
430
+ prompt,
431
sampling_params=sampling_params,
432
use_tqdm=use_tqdm,
433
lora_request=lora_request,
@@ -494,8 +497,8 @@ class LLM:
497
@overload
498
def encode(
499
self,
497
- inputs: Union[PromptInputs, Sequence[PromptInputs]],
498
- /, # We may enable `inputs` keyword after removing the old API
500
+ prompts: Union[PromptType, Sequence[PromptType]],
501
+ /,
502
*,
503
pooling_params: Optional[Union[PoolingParams,
504
Sequence[PoolingParams]]] = None,
@@ -512,7 +515,7 @@ class LLM:
515
)
516
def encode(
517
self,
515
- prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
518
+ prompts: Union[Union[PromptType, Sequence[PromptType]],
519
Optional[Union[str, List[str]]]] = None,
520
pooling_params: Optional[Union[PoolingParams,
521
Sequence[PoolingParams]]] = None,
@@ -528,9 +531,9 @@ class LLM:
531
into a single list and pass it to this method.
532
533
Args:
531
- inputs: The inputs to the LLM. You may pass a sequence of inputs for
532
- batch inference. See :class:`~vllm.inputs.PromptInputs`
533
- for more details about the format of each input.
534
+ prompts: The prompts to the LLM. You may pass a sequence of prompts
535
+ for batch inference. See :class:`~vllm.inputs.PromptType`
536
+ for more details about the format of each prompts.
537
pooling_params: The pooling parameters for pooling. If None, we
538
use the default pooling parameters.
539
use_tqdm: Whether to use tqdm to display the progress bar.
@@ -553,19 +556,20 @@ class LLM:
556
)
557
558
if prompt_token_ids is not None:
556
- inputs = self._convert_v1_inputs(
559
+ parsed_prompts = self._convert_v1_inputs(
560
prompts=cast(Optional[Union[str, List[str]]], prompts),
561
prompt_token_ids=prompt_token_ids,
562
)
563
else:
561
- inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
564
+ parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
565
+ prompts)
566
567
if pooling_params is None:
568
# Use default pooling params.
569
pooling_params = PoolingParams()
570
571
self._validate_and_add_requests(
568
- inputs=inputs,
572
+ prompts=parsed_prompts,
573
params=pooling_params,
574
lora_request=lora_request,
575
prompt_adapter_request=prompt_adapter_request,
@@ -609,9 +613,9 @@ class LLM:
613
raise ValueError("Either prompts or prompt_token_ids must be "
614
"provided.")
615
612
- inputs: List[PromptInputs] = []
616
+ parsed_prompts: List[PromptType] = []
617
for i in range(num_requests):
614
- item: PromptInputs
618
+ item: PromptType
619
620
if prompts is not None:
621
item = TextPrompt(prompt=prompts[i])
@@ -620,24 +624,24 @@ class LLM:
624
else:
625
raise AssertionError
626
623
- inputs.append(item)
627
+ parsed_prompts.append(item)
628
625
- return inputs
629
+ return parsed_prompts
630
631
def _validate_and_add_requests(
632
self,
629
- inputs: Union[PromptInputs, Sequence[PromptInputs]],
633
+ prompts: Union[PromptType, Sequence[PromptType]],
634
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
635
Sequence[PoolingParams]],
636
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
637
prompt_adapter_request: Optional[PromptAdapterRequest],
638
guided_options: Optional[GuidedDecodingRequest] = None,
639
) -> None:
636
- if isinstance(inputs, (str, dict)):
640
+ if isinstance(prompts, (str, dict)):
641
# Convert a single prompt to a list.
638
- inputs = [inputs]
642
+ prompts = [prompts]
643
640
- num_requests = len(inputs)
644
+ num_requests = len(prompts)
645
if isinstance(params, list) and len(params) != num_requests:
646
raise ValueError("The lengths of prompts and params "
647
"must be the same.")
@@ -654,9 +658,9 @@ class LLM:
658
sp.output_kind = RequestOutputKind.FINAL_ONLY
659
660
# Add requests to the engine.
657
- for i, request_inputs in enumerate(inputs):
661
+ for i, prompt in enumerate(prompts):
662
self._add_request(
659
- request_inputs,
663
+ prompt,
664
params[i] if isinstance(params, Sequence) else params,
665
lora_request=lora_request[i] if isinstance(
666
lora_request, Sequence) else lora_request,
@@ -665,7 +669,7 @@ class LLM:
669
670
def _add_request(
671
self,
668
- inputs: PromptInputs,
672
+ prompt: PromptType,
673
params: Union[SamplingParams, PoolingParams],
674
lora_request: Optional[LoRARequest] = None,
675
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -673,7 +677,7 @@ class LLM:
677
request_id = str(next(self.request_counter))
678
self.llm_engine.add_request(
679
request_id,
676
- inputs,
680
+ prompt,
681
params,
682
lora_request=lora_request,
683
prompt_adapter_request=prompt_adapter_request,
vllm/inputs/__init__.py
+3
-3
@@ -1,5 +1,5 @@
1
from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2
- LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
2
+ LLMInputs, PromptType, SingletonPrompt, TextPrompt,
3
TokensPrompt, build_explicit_enc_dec_prompt,
4
to_enc_dec_tuple_list, zip_enc_dec_prompts)
5
from .registry import InputContext, InputRegistry
@@ -16,8 +16,8 @@ See also:
16
__all__ = [
17
"TextPrompt",
18
"TokensPrompt",
19
- "PromptInputs",
20
- "SingletonPromptInputs",
19
+ "PromptType",
20
+ "SingletonPrompt",
21
"ExplicitEncoderDecoderPrompt",
22
"LLMInputs",
23
"EncoderDecoderLLMInputs",
vllm/inputs/data.py
+11
-15
@@ -33,7 +33,7 @@ class TokensPrompt(TypedDict):
33
"""
34
35
36
-SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
36
+SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
37
"""
38
Set of possible schemas for a single LLM input:
39
@@ -46,7 +46,7 @@ which may be utilized for encoder/decoder models when
46
the user desires to express both the encoder & decoder
47
prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
49
-A prompt of type :class:`SingletonPromptInputs` may be employed
49
+A prompt of type :class:`SingletonPromptType` may be employed
50
as (1) input to a decoder-only model, (2) input to
51
the encoder of an encoder/decoder model, in the scenario
52
where the decoder-prompt is not specified explicitly, or
@@ -55,12 +55,12 @@ more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55
"""
56
57
_T1_co = TypeVar("_T1_co",
58
- bound=SingletonPromptInputs,
59
- default=SingletonPromptInputs,
58
+ bound=SingletonPrompt,
59
+ default=SingletonPrompt,
60
covariant=True)
61
_T2_co = TypeVar("_T2_co",
62
- bound=SingletonPromptInputs,
63
- default=SingletonPromptInputs,
62
+ bound=SingletonPrompt,
63
+ default=SingletonPrompt,
64
covariant=True)
65
66
@@ -72,7 +72,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
72
73
The encoder and decoder prompts, respectively,
74
may formatted according to any of the
75
- :class:`SingletonPromptInputs` schemas, and are not
75
+ :class:`SingletonPromptType` schemas, and are not
76
required to have the same schema.
77
78
Only the encoder prompt may have multi-modal data.
@@ -81,7 +81,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
81
be used as an input to a decoder-only model,
82
and that the `encoder_prompt` and `decoder_prompt`
83
fields of this data structure themselves must be
84
- :class:`SingletonPromptInputs` instances.
84
+ :class:`SingletonPromptType` instances.
85
"""
86
87
encoder_prompt: _T1_co
@@ -89,7 +89,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
89
decoder_prompt: Optional[_T2_co]
90
91
92
-PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
92
+PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
93
"""
94
Set of possible schemas for an LLM input, including
95
both decoder-only and encoder/decoder input types:
@@ -140,12 +140,8 @@ class EncoderDecoderLLMInputs(LLMInputs):
140
"""
141
142
143
-_T1 = TypeVar("_T1",
144
- bound=SingletonPromptInputs,
145
- default=SingletonPromptInputs)
146
-_T2 = TypeVar("_T2",
147
- bound=SingletonPromptInputs,
148
- default=SingletonPromptInputs)
143
+_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
144
+_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
145
146
147
def build_explicit_enc_dec_prompt(
vllm/inputs/parse.py
+11
-11
@@ -5,7 +5,7 @@ from typing_extensions import TypeIs
5
from vllm.utils import is_list_of
6
7
from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8
- LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
8
+ LLMInputs, PromptType, SingletonPrompt, TextPrompt,
9
TokensPrompt)
10
11
@@ -81,23 +81,23 @@ class ParsedTokensPrompt(TypedDict):
81
82
83
def parse_singleton_prompt(
84
- inputs: SingletonPromptInputs,
84
+ prompt: SingletonPrompt,
85
) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86
- if isinstance(inputs, str):
87
- return ParsedStrPrompt(type="str", content=inputs)
88
- elif isinstance(inputs, dict):
89
- if "prompt_token_ids" in inputs:
86
+ if isinstance(prompt, str):
87
+ return ParsedStrPrompt(type="str", content=prompt)
88
+ elif isinstance(prompt, dict):
89
+ if "prompt_token_ids" in prompt:
90
return ParsedTokensPrompt(type="tokens",
91
- content=inputs) # type: ignore
92
- elif "prompt" in inputs:
93
- return ParsedTextPrompt(type="text", content=inputs)
91
+ content=prompt) # type: ignore
92
+ elif "prompt" in prompt:
93
+ return ParsedTextPrompt(type="text", content=prompt)
94
95
raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98
def is_explicit_encoder_decoder_prompt(
99
- inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100
- return isinstance(inputs, dict) and "encoder_prompt" in inputs
99
+ prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100
+ return isinstance(prompt, dict) and "encoder_prompt" in prompt
101
102
103
def is_valid_encoder_decoder_llm_inputs(
vllm/inputs/preprocess.py
+43
-43
@@ -9,8 +9,8 @@ from vllm.lora.request import LoRARequest
9
from vllm.prompt_adapter.request import PromptAdapterRequest
10
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
12
-from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
13
- SingletonPromptInputs)
12
+from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
13
+ SingletonPrompt)
14
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
15
16
if TYPE_CHECKING:
@@ -206,7 +206,7 @@ class InputPreprocessor:
206
207
def _extract_prompt_components(
208
self,
209
- inputs: SingletonPromptInputs,
209
+ prompt: SingletonPrompt,
210
request_id: str,
211
lora_request: Optional[LoRARequest] = None,
212
) -> PromptComponents:
@@ -216,7 +216,7 @@ class InputPreprocessor:
216
Arguments:
217
218
* request_id
219
- * inputs: single encoder or decoder input prompt
219
+ * prompt: single encoder or decoder input prompt
220
* lora_request: this is only valid for decoder prompts
221
222
Returns:
@@ -226,24 +226,24 @@ class InputPreprocessor:
226
* multi_modal_data
227
'''
228
229
- parsed = parse_singleton_prompt(inputs)
229
+ parsed = parse_singleton_prompt(prompt)
230
231
if parsed["type"] == "str":
232
- prompt = parsed["content"]
232
+ prompt_text = parsed["content"]
233
prompt_token_ids = self._tokenize_prompt(
234
- prompt,
234
+ prompt_text,
235
request_id=request_id,
236
lora_request=lora_request,
237
)
238
multi_modal_data = None
239
elif parsed["type"] == "tokens":
240
- prompt = None
240
+ prompt_text = None
241
prompt_token_ids = parsed["content"]["prompt_token_ids"]
242
multi_modal_data = parsed["content"].get("multi_modal_data")
243
elif parsed["type"] == "text":
244
- prompt = parsed["content"]["prompt"]
244
+ prompt_text = parsed["content"]["prompt"]
245
prompt_token_ids = self._tokenize_prompt(
246
- prompt,
246
+ prompt_text,
247
request_id=request_id,
248
lora_request=lora_request,
249
)
@@ -251,33 +251,33 @@ class InputPreprocessor:
251
else:
252
assert_never(parsed)
253
254
- return prompt, prompt_token_ids, multi_modal_data
254
+ return prompt_text, prompt_token_ids, multi_modal_data
255
256
async def _extract_prompt_components_async(
257
self,
258
- inputs: SingletonPromptInputs,
258
+ prompt: SingletonPrompt,
259
request_id: str,
260
lora_request: Optional[LoRARequest] = None,
261
) -> PromptComponents:
262
"""Async version of :meth:`_extract_prompt_components`."""
263
- parsed = parse_singleton_prompt(inputs)
263
+ parsed = parse_singleton_prompt(prompt)
264
265
if parsed["type"] == "str":
266
- prompt = parsed["content"]
266
+ prompt_text = parsed["content"]
267
prompt_token_ids = await self._tokenize_prompt_async(
268
- prompt,
268
+ prompt_text,
269
request_id=request_id,
270
lora_request=lora_request,
271
)
272
multi_modal_data = None
273
elif parsed["type"] == "tokens":
274
- prompt = None
274
+ prompt_text = None
275
prompt_token_ids = parsed["content"]["prompt_token_ids"]
276
multi_modal_data = parsed["content"].get("multi_modal_data")
277
elif parsed["type"] == "text":
278
- prompt = parsed["content"]["prompt"]
278
+ prompt_text = parsed["content"]["prompt"]
279
prompt_token_ids = await self._tokenize_prompt_async(
280
- prompt,
280
+ prompt_text,
281
request_id=request_id,
282
lora_request=lora_request,
283
)
@@ -285,7 +285,7 @@ class InputPreprocessor:
285
else:
286
assert_never(parsed)
287
288
- return prompt, prompt_token_ids, multi_modal_data
288
+ return prompt_text, prompt_token_ids, multi_modal_data
289
290
def _build_enc_dec_llm_inputs(
291
self,
@@ -311,7 +311,7 @@ class InputPreprocessor:
311
312
def _process_encoder_decoder_prompt(
313
self,
314
- inputs: PromptInputs,
314
+ prompt: PromptType,
315
request_id: str,
316
) -> EncoderDecoderLLMInputs:
317
'''
@@ -339,7 +339,7 @@ class InputPreprocessor:
339
340
Arguments:
341
342
- * inputs: an input prompt
342
+ * prompt: an input prompt
343
* request_id
344
345
Returns:
@@ -350,13 +350,13 @@ class InputPreprocessor:
350
encoder_comps: PromptComponents
351
decoder_comps: DecoderPromptComponents
352
353
- if is_explicit_encoder_decoder_prompt(inputs):
353
+ if is_explicit_encoder_decoder_prompt(prompt):
354
encoder_comps = self._extract_prompt_components(
355
- inputs["encoder_prompt"],
355
+ prompt["encoder_prompt"],
356
request_id=request_id,
357
)
358
359
- if (decoder_input := inputs["decoder_prompt"]) is None:
359
+ if (decoder_input := prompt["decoder_prompt"]) is None:
360
decoder_comps = None, None, None
361
else:
362
decoder_comps = self._extract_prompt_components(
@@ -365,7 +365,7 @@ class InputPreprocessor:
365
)
366
else:
367
encoder_comps = self._extract_prompt_components(
368
- inputs,
368
+ prompt,
369
request_id=request_id,
370
)
371
@@ -375,20 +375,20 @@ class InputPreprocessor:
375
376
async def _process_encoder_decoder_prompt_async(
377
self,
378
- inputs: PromptInputs,
378
+ prompt: PromptType,
379
request_id: str,
380
) -> EncoderDecoderLLMInputs:
381
"""Async version of :meth:`_process_encoder_decoder_prompt`."""
382
encoder_comps: PromptComponents
383
decoder_comps: DecoderPromptComponents
384
385
- if is_explicit_encoder_decoder_prompt(inputs):
385
+ if is_explicit_encoder_decoder_prompt(prompt):
386
encoder_task = self._extract_prompt_components_async(
387
- inputs["encoder_prompt"],
387
+ prompt["encoder_prompt"],
388
request_id=request_id,
389
)
390
391
- if (decoder_input := inputs["decoder_prompt"]) is None:
391
+ if (decoder_input := prompt["decoder_prompt"]) is None:
392
encoder_comps = await encoder_task
393
decoder_comps = None, None, None
394
else:
@@ -401,7 +401,7 @@ class InputPreprocessor:
401
encoder_task, decoder_task)
402
else:
403
encoder_comps = await self._extract_prompt_components_async(
404
- inputs,
404
+ prompt,
405
request_id=request_id,
406
)
407
@@ -425,7 +425,7 @@ class InputPreprocessor:
425
426
def _process_decoder_only_prompt(
427
self,
428
- inputs: SingletonPromptInputs,
428
+ prompt: SingletonPrompt,
429
request_id: str,
430
lora_request: Optional[LoRARequest] = None,
431
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -436,7 +436,7 @@ class InputPreprocessor:
436
437
Arguments:
438
439
- * inputs: input prompt
439
+ * prompt: input prompt
440
* request_id
441
* lora_request
442
* prompt_adapter_request
@@ -447,7 +447,7 @@ class InputPreprocessor:
447
'''
448
449
prompt_comps = self._extract_prompt_components(
450
- inputs,
450
+ prompt,
451
request_id=request_id,
452
lora_request=lora_request,
453
)
@@ -459,14 +459,14 @@ class InputPreprocessor:
459
460
async def _process_decoder_only_prompt_async(
461
self,
462
- inputs: SingletonPromptInputs,
462
+ prompt: SingletonPrompt,
463
request_id: str,
464
lora_request: Optional[LoRARequest] = None,
465
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
466
) -> LLMInputs:
467
"""Async version of :meth:`_process_decoder_only_prompt`."""
468
prompt_comps = await self._extract_prompt_components_async(
469
- inputs,
469
+ prompt,
470
request_id=request_id,
471
lora_request=lora_request,
472
)
@@ -478,7 +478,7 @@ class InputPreprocessor:
478
479
def preprocess(
480
self,
481
- inputs: PromptInputs,
481
+ prompt: PromptType,
482
request_id: str,
483
lora_request: Optional[LoRARequest] = None,
484
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -488,17 +488,17 @@ class InputPreprocessor:
488
# Encoder-decoder model requires special mapping of
489
# input prompts to encoder & decoder
490
return self._process_encoder_decoder_prompt(
491
- inputs,
491
+ prompt,
492
request_id=request_id,
493
)
494
495
- if is_explicit_encoder_decoder_prompt(inputs):
495
+ if is_explicit_encoder_decoder_prompt(prompt):
496
raise ValueError("Cannot pass encoder-decoder prompt "
497
"to decoder-only models")
498
499
# Decoder-only operation
500
return self._process_decoder_only_prompt(
501
- inputs,
501
+ prompt,
502
request_id=request_id,
503
lora_request=lora_request,
504
prompt_adapter_request=prompt_adapter_request,
@@ -506,7 +506,7 @@ class InputPreprocessor:
506
507
async def preprocess_async(
508
self,
509
- inputs: PromptInputs,
509
+ prompt: PromptType,
510
request_id: str,
511
lora_request: Optional[LoRARequest] = None,
512
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -516,17 +516,17 @@ class InputPreprocessor:
516
# Encoder-decoder model requires special mapping of
517
# input prompts to encoder & decoder
518
return await self._process_encoder_decoder_prompt_async(
519
- inputs,
519
+ prompt,
520
request_id=request_id,
521
)
522
523
- if is_explicit_encoder_decoder_prompt(inputs):
523
+ if is_explicit_encoder_decoder_prompt(prompt):
524
raise ValueError("Cannot pass encoder-decoder prompt "
525
"to decoder-only models")
526
527
# Decoder-only operation
528
return await self._process_decoder_only_prompt_async(
529
- inputs,
529
+ prompt,
530
request_id=request_id,
531
lora_request=lora_request,
532
prompt_adapter_request=prompt_adapter_request,