A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] rename`PromptInputs` and `inputs` (#8876)
Cyrus Leung committed
Sep 27, 2024 at 11:35 UTC
3b00b9c26c91e9f9ada12975b613555698054e39
21 files changed
+440
-248
benchmarks/benchmark_latency.py
+4
-4
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12
from vllm import LLM, SamplingParams
13
from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14
-from vllm.inputs import PromptInputs
14
+from vllm.inputs import PromptType
15
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61
dummy_prompt_token_ids = np.random.randint(10000,
62
size=(args.batch_size,
63
args.input_len))
64
- dummy_inputs: List[PromptInputs] = [{
64
+ dummy_prompts: List[PromptType] = [{
65
"prompt_token_ids": batch
66
} for batch in dummy_prompt_token_ids.tolist()]
67
@@ -74,13 +74,13 @@ def main(args: argparse.Namespace):
74
],
75
on_trace_ready=torch.profiler.tensorboard_trace_handler(
76
str(profile_dir))) as p:
77
- llm.generate(dummy_inputs,
77
+ llm.generate(dummy_prompts,
78
sampling_params=sampling_params,
79
use_tqdm=False)
80
print(p.key_averages())
81
else:
82
start_time = time.perf_counter()
83
- llm.generate(dummy_inputs,
83
+ llm.generate(dummy_prompts,
84
sampling_params=sampling_params,
85
use_tqdm=False)
86
end_time = time.perf_counter()
docs/source/dev/multimodal/multimodal_index.rst
+1
-1
@@ -8,7 +8,7 @@ Multi-Modality
8
vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10
Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11
-via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
11
+via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
12
13
Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14
by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1
-1
@@ -1,7 +1,7 @@
1
LLM Inputs
2
==========
3
4
-.. autodata:: vllm.inputs.PromptInputs
4
+.. autodata:: vllm.inputs.PromptType
5
6
.. autoclass:: vllm.inputs.TextPrompt
7
:show-inheritance:
docs/source/models/vlm.rst
+1
-1
@@ -27,7 +27,7 @@ The :class:`~vllm.LLM` class can be instantiated in much the same way as languag
27
We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28
the above snippet. Specifically, ``image_feature_size`` can no longer be specified as we now calculate that internally for each model.
29
30
-To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
30
+To pass an image to the model, note the following in :class:`vllm.inputs.PromptType`:
31
32
* ``prompt``: The prompt should follow the format that is documented on HuggingFace.
33
* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/async_engine/test_async_llm_engine.py
+5
-3
@@ -86,17 +86,19 @@ class MockAsyncLLMEngine(AsyncLLMEngine):
86
87
@pytest.mark.asyncio
88
async def test_new_requests_event():
89
+ params = SamplingParams()
90
+
91
engine = MockAsyncLLMEngine()
92
engine.start_background_loop()
93
await asyncio.sleep(0.01)
94
assert engine.engine.step_calls == 0
95
94
- await engine.add_request("1", "", None)
96
+ await engine.add_request("1", "", params)
97
await asyncio.sleep(0.01)
98
assert engine.engine.add_request_calls == 1
99
assert engine.engine.step_calls == 1
100
99
- await engine.add_request("2", "", None)
101
+ await engine.add_request("2", "", params)
102
engine.engine.generate("2")
103
await asyncio.sleep(0)
104
await asyncio.sleep(0)
@@ -111,7 +113,7 @@ async def test_new_requests_event():
113
await asyncio.sleep(0.001)
114
assert engine.engine.step_calls == old_step_calls
115
114
- await engine.add_request("3", "", None)
116
+ await engine.add_request("3", "", params)
117
await asyncio.sleep(0.01)
118
assert engine.engine.add_request_calls == 3
119
assert engine.engine.step_calls == old_step_calls + 1
tests/entrypoints/llm/test_encode.py
-34
@@ -49,21 +49,6 @@ def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
49
assert [o.outputs for o in o1] == [o.outputs for o in o2]
50
51
52
-@pytest.mark.skip_global_cleanup
53
-@pytest.mark.parametrize('prompt', PROMPTS)
54
-def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
55
- pooling_params = PoolingParams()
56
-
57
- with pytest.warns(DeprecationWarning, match="'prompts'"):
58
- v1_output = llm.encode(prompts=prompt, pooling_params=pooling_params)
59
-
60
- v2_output = llm.encode(prompt, pooling_params=pooling_params)
61
- assert_outputs_equal(v1_output, v2_output)
62
-
63
- v2_output = llm.encode({"prompt": prompt}, pooling_params=pooling_params)
64
- assert_outputs_equal(v1_output, v2_output)
65
-
66
-
52
@pytest.mark.skip_global_cleanup
53
@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
54
def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -79,25 +64,6 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
64
assert_outputs_equal(v1_output, v2_output)
65
66
82
-@pytest.mark.skip_global_cleanup
83
-def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84
- pooling_params = PoolingParams()
85
-
86
- with pytest.warns(DeprecationWarning, match="'prompts'"):
87
- v1_output = llm.encode(prompts=PROMPTS, pooling_params=pooling_params)
88
-
89
- v2_output = llm.encode(PROMPTS, pooling_params=pooling_params)
90
- assert_outputs_equal(v1_output, v2_output)
91
-
92
- v2_output = llm.encode(
93
- [{
94
- "prompt": p
95
- } for p in PROMPTS],
96
- pooling_params=pooling_params,
97
- )
98
- assert_outputs_equal(v1_output, v2_output)
99
-
100
-
67
@pytest.mark.skip_global_cleanup
68
def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
69
pooling_params = PoolingParams()
tests/entrypoints/llm/test_generate.py
-37
@@ -47,23 +47,6 @@ def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
47
assert [o.outputs for o in o1] == [o.outputs for o in o2]
48
49
50
-@pytest.mark.skip_global_cleanup
51
-@pytest.mark.parametrize('prompt', PROMPTS)
52
-def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
53
- sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
54
-
55
- with pytest.warns(DeprecationWarning, match="'prompts'"):
56
- v1_output = llm.generate(prompts=prompt,
57
- sampling_params=sampling_params)
58
-
59
- v2_output = llm.generate(prompt, sampling_params=sampling_params)
60
- assert_outputs_equal(v1_output, v2_output)
61
-
62
- v2_output = llm.generate({"prompt": prompt},
63
- sampling_params=sampling_params)
64
- assert_outputs_equal(v1_output, v2_output)
65
-
66
-
50
@pytest.mark.skip_global_cleanup
51
@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
52
def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -79,26 +62,6 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
62
assert_outputs_equal(v1_output, v2_output)
63
64
82
-@pytest.mark.skip_global_cleanup
83
-def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84
- sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
85
-
86
- with pytest.warns(DeprecationWarning, match="'prompts'"):
87
- v1_output = llm.generate(prompts=PROMPTS,
88
- sampling_params=sampling_params)
89
-
90
- v2_output = llm.generate(PROMPTS, sampling_params=sampling_params)
91
- assert_outputs_equal(v1_output, v2_output)
92
-
93
- v2_output = llm.generate(
94
- [{
95
- "prompt": p
96
- } for p in PROMPTS],
97
- sampling_params=sampling_params,
98
- )
99
- assert_outputs_equal(v1_output, v2_output)
100
-
101
-
65
@pytest.mark.skip_global_cleanup
66
def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
67
sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
tests/mq_llm_engine/test_error_handling.py
+6
-6
@@ -61,7 +61,7 @@ async def test_evil_forward(tmp_socket):
61
62
# Throws an error in first forward pass.
63
with pytest.raises(RAISED_ERROR):
64
- async for _ in client.generate(inputs="Hello my name is",
64
+ async for _ in client.generate(prompt="Hello my name is",
65
sampling_params=SamplingParams(),
66
request_id=uuid.uuid4()):
67
pass
@@ -69,7 +69,7 @@ async def test_evil_forward(tmp_socket):
69
70
# Engine is errored, should get ENGINE_DEAD_ERROR.
71
with pytest.raises(MQEngineDeadError):
72
- async for _ in client.generate(inputs="Hello my name is",
72
+ async for _ in client.generate(prompt="Hello my name is",
73
sampling_params=SamplingParams(),
74
request_id=uuid.uuid4()):
75
pass
@@ -118,7 +118,7 @@ async def test_failed_health_check(tmp_socket):
118
119
# Generate call should throw ENGINE_DEAD_ERROR
120
with pytest.raises(MQEngineDeadError):
121
- async for _ in client.generate(inputs="Hello my name is",
121
+ async for _ in client.generate(prompt="Hello my name is",
122
sampling_params=SamplingParams(),
123
request_id=uuid.uuid4()):
124
pass
@@ -160,7 +160,7 @@ async def test_failed_abort(tmp_socket):
160
# with reference to the original KeyError("foo")
161
with pytest.raises(MQEngineDeadError) as execinfo:
162
async for _ in client.generate(
163
- inputs="Hello my name is",
163
+ prompt="Hello my name is",
164
sampling_params=SamplingParams(max_tokens=10),
165
request_id=uuid.uuid4()):
166
pass
@@ -183,7 +183,7 @@ async def test_bad_request(tmp_socket):
183
184
# Invalid request should fail, but not crash the server.
185
with pytest.raises(ValueError):
186
- async for _ in client.generate(inputs="Hello my name is",
186
+ async for _ in client.generate(prompt="Hello my name is",
187
sampling_params=SamplingParams(),
188
request_id="abcd-1",
189
lora_request=LoRARequest(
@@ -192,7 +192,7 @@ async def test_bad_request(tmp_socket):
192
pass
193
194
# This request should be okay.
195
- async for _ in client.generate(inputs="Hello my name is",
195
+ async for _ in client.generate(prompt="Hello my name is",
196
sampling_params=SamplingParams(),
197
request_id="abcd-2"):
198
pass
tests/mq_llm_engine/utils.py
+1
-1
@@ -20,7 +20,7 @@ async def generate(
20
count = 0
21
async for out in client.generate(
22
request_id=request_id,
23
- inputs="Hello my name is Robert and",
23
+ prompt="Hello my name is Robert and",
24
sampling_params=SamplingParams(max_tokens=num_tokens,
25
temperature=0)):
26
vllm/__init__.py
+2
-2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
-from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
8
+from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11
EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19
"__version_tuple__",
20
"LLM",
21
"ModelRegistry",
22
- "PromptInputs",
22
+ "PromptType",
23
"TextPrompt",
24
"TokensPrompt",
25
"SamplingParams",
vllm/engine/async_llm_engine.py
+92
-18
@@ -2,8 +2,8 @@ import asyncio
2
import time
3
import weakref
4
from functools import partial
5
-from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6
- Mapping, Optional, Set, Tuple, Type, Union)
5
+from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
6
+ List, Mapping, Optional, Set, Tuple, Type, Union, overload)
7
from weakref import ReferenceType
8
9
import vllm.envs as envs
@@ -17,7 +17,7 @@ from vllm.engine.metrics_types import StatLoggerBase
17
from vllm.executor.executor_base import ExecutorAsyncBase
18
from vllm.executor.gpu_executor import GPUExecutorAsync
19
from vllm.executor.ray_utils import initialize_ray_cluster
20
-from vllm.inputs import PromptInputs
20
+from vllm.inputs import PromptType
21
from vllm.logger import init_logger
22
from vllm.lora.request import LoRARequest
23
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -28,7 +28,7 @@ from vllm.sampling_params import SamplingParams
28
from vllm.sequence import ExecuteModelRequest
29
from vllm.transformers_utils.tokenizer import AnyTokenizer
30
from vllm.usage.usage_lib import UsageContext
31
-from vllm.utils import weak_bind
31
+from vllm.utils import deprecate_kwargs, weak_bind
32
33
logger = init_logger(__name__)
34
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -402,17 +402,54 @@ class _AsyncLLMEngine(LLMEngine):
402
"""Stop the remote worker execution loop."""
403
await self.model_executor.stop_remote_worker_execution_loop_async()
404
405
+ @overload # DEPRECATED
406
async def add_request_async(
407
self,
408
request_id: str,
408
- inputs: PromptInputs,
409
+ *,
410
+ inputs: PromptType,
411
params: Union[SamplingParams, PoolingParams],
412
arrival_time: Optional[float] = None,
413
lora_request: Optional[LoRARequest] = None,
414
trace_headers: Optional[Mapping[str, str]] = None,
415
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
416
+ ) -> None:
417
+ ...
418
+
419
+ @overload
420
+ async def add_request_async(
421
+ self,
422
+ request_id: str,
423
+ prompt: PromptType,
424
+ params: Union[SamplingParams, PoolingParams],
425
+ arrival_time: Optional[float] = None,
426
+ lora_request: Optional[LoRARequest] = None,
427
+ trace_headers: Optional[Mapping[str, str]] = None,
428
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
429
+ ) -> None:
430
+ ...
431
+
432
+ @deprecate_kwargs(
433
+ "inputs",
434
+ additional_message="Please use the 'prompt' parameter instead.",
435
+ )
436
+ async def add_request_async(
437
+ self,
438
+ request_id: str,
439
+ prompt: Optional[PromptType] = None,
440
+ params: Optional[Union[SamplingParams, PoolingParams]] = None,
441
+ arrival_time: Optional[float] = None,
442
+ lora_request: Optional[LoRARequest] = None,
443
+ trace_headers: Optional[Mapping[str, str]] = None,
444
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
445
+ *,
446
+ inputs: Optional[PromptType] = None, # DEPRECATED
447
) -> None:
448
"""Async version of :meth:`add_request`."""
449
+ if inputs is not None:
450
+ prompt = inputs
451
+ assert prompt is not None and params is not None
452
+
453
if lora_request is not None and not self.lora_config:
454
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
455
"not enabled!")
@@ -420,7 +457,7 @@ class _AsyncLLMEngine(LLMEngine):
457
arrival_time = time.time()
458
459
preprocessed_inputs = await self.input_preprocessor.preprocess_async(
423
- inputs,
460
+ prompt,
461
request_id=request_id,
462
lora_request=lora_request,
463
prompt_adapter_request=prompt_adapter_request,
@@ -774,16 +811,55 @@ class AsyncLLMEngine:
811
812
# This method does not need to be async, but kept that way
813
# for backwards compatibility.
777
- async def add_request(
814
+ @overload # DEPRECATED
815
+ def add_request(
816
self,
817
request_id: str,
780
- inputs: PromptInputs,
818
+ *,
819
+ inputs: PromptType,
820
params: Union[SamplingParams, PoolingParams],
821
arrival_time: Optional[float] = None,
822
lora_request: Optional[LoRARequest] = None,
823
trace_headers: Optional[Mapping[str, str]] = None,
785
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
824
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
825
+ ) -> Coroutine[None, None, AsyncGenerator[Union[
826
+ RequestOutput, EmbeddingRequestOutput], None]]:
827
+ ...
828
+
829
+ @overload
830
+ def add_request(
831
+ self,
832
+ request_id: str,
833
+ prompt: PromptType,
834
+ params: Union[SamplingParams, PoolingParams],
835
+ arrival_time: Optional[float] = None,
836
+ lora_request: Optional[LoRARequest] = None,
837
+ trace_headers: Optional[Mapping[str, str]] = None,
838
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
839
+ ) -> Coroutine[None, None, AsyncGenerator[Union[
840
+ RequestOutput, EmbeddingRequestOutput], None]]:
841
+ ...
842
+
843
+ @deprecate_kwargs(
844
+ "inputs",
845
+ additional_message="Please use the 'prompt' parameter instead.",
846
+ )
847
+ async def add_request(
848
+ self,
849
+ request_id: str,
850
+ prompt: Optional[PromptType] = None,
851
+ params: Optional[Union[SamplingParams, PoolingParams]] = None,
852
+ arrival_time: Optional[float] = None,
853
+ lora_request: Optional[LoRARequest] = None,
854
+ trace_headers: Optional[Mapping[str, str]] = None,
855
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
856
+ *,
857
+ inputs: Optional[PromptType] = None, # DEPRECATED
858
) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
859
+ if inputs is not None:
860
+ prompt = inputs
861
+ assert prompt is not None and params is not None
862
+
863
if not self.is_running:
864
if self.start_engine_loop:
865
self.start_background_loop()
@@ -797,7 +873,7 @@ class AsyncLLMEngine:
873
stream = self._request_tracker.add_request(
874
request_id,
875
verbose=self.log_requests,
800
- inputs=inputs,
876
+ prompt=prompt,
877
params=params,
878
arrival_time=arrival_time or time.time(),
879
lora_request=lora_request,
@@ -808,7 +884,7 @@ class AsyncLLMEngine:
884
885
async def generate(
886
self,
811
- inputs: PromptInputs,
887
+ prompt: PromptType,
888
sampling_params: SamplingParams,
889
request_id: str,
890
lora_request: Optional[LoRARequest] = None,
@@ -822,8 +898,7 @@ class AsyncLLMEngine:
898
from the LLMEngine to the caller.
899
900
Args:
825
- inputs: The inputs to the LLM. See
826
- :class:`~vllm.inputs.PromptInputs`
901
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
902
for more details about the format of each input.
903
sampling_params: The sampling parameters of the request.
904
request_id: The unique id of the request.
@@ -881,7 +956,7 @@ class AsyncLLMEngine:
956
"""
957
async for output in await self.add_request(
958
request_id,
884
- inputs,
959
+ prompt,
960
sampling_params,
961
lora_request=lora_request,
962
trace_headers=trace_headers,
@@ -891,7 +966,7 @@ class AsyncLLMEngine:
966
967
async def encode(
968
self,
894
- inputs: PromptInputs,
969
+ prompt: PromptType,
970
pooling_params: PoolingParams,
971
request_id: str,
972
lora_request: Optional[LoRARequest] = None,
@@ -904,8 +979,7 @@ class AsyncLLMEngine:
979
from the LLMEngine to the caller.
980
981
Args:
907
- inputs: The inputs to the LLM. See
908
- :class:`~vllm.inputs.PromptInputs`
982
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
983
for more details about the format of each input.
984
pooling_params: The pooling parameters of the request.
985
request_id: The unique id of the request.
@@ -959,7 +1033,7 @@ class AsyncLLMEngine:
1033
"""
1034
async for output in await self.add_request(
1035
request_id,
962
- inputs,
1036
+ prompt,
1037
pooling_params,
1038
lora_request=lora_request,
1039
trace_headers=trace_headers,
vllm/engine/llm_engine.py
+45
-7
@@ -6,7 +6,7 @@ from functools import partial
6
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7
Iterable, List, Mapping, NamedTuple, Optional)
8
from typing import Sequence as GenericSequence
9
-from typing import Set, Type, Union
9
+from typing import Set, Type, Union, overload
10
11
import torch
12
from typing_extensions import TypeVar
@@ -29,7 +29,7 @@ from vllm.executor.executor_base import ExecutorBase
29
from vllm.executor.gpu_executor import GPUExecutor
30
from vllm.executor.ray_utils import initialize_ray_cluster
31
from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32
- InputRegistry, LLMInputs, PromptInputs)
32
+ InputRegistry, LLMInputs, PromptType)
33
from vllm.inputs.preprocess import InputPreprocessor
34
from vllm.logger import init_logger
35
from vllm.lora.request import LoRARequest
@@ -51,7 +51,7 @@ from vllm.transformers_utils.tokenizer_group import (
51
BaseTokenizerGroup, init_tokenizer_from_configs)
52
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
53
usage_message)
54
-from vllm.utils import Counter, Device, weak_bind
54
+from vllm.utils import Counter, Device, deprecate_kwargs, weak_bind
55
from vllm.version import __version__ as VLLM_VERSION
56
57
logger = init_logger(__name__)
@@ -689,16 +689,51 @@ class LLMEngine:
689
def stop_remote_worker_execution_loop(self) -> None:
690
self.model_executor.stop_remote_worker_execution_loop()
691
692
+ @overload # DEPRECATED
693
def add_request(
694
self,
695
request_id: str,
695
- inputs: PromptInputs,
696
+ *,
697
+ inputs: PromptType,
698
params: Union[SamplingParams, PoolingParams],
699
arrival_time: Optional[float] = None,
700
lora_request: Optional[LoRARequest] = None,
701
trace_headers: Optional[Mapping[str, str]] = None,
702
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
703
priority: int = 0,
704
+ ) -> None:
705
+ ...
706
+
707
+ @overload
708
+ def add_request(
709
+ self,
710
+ request_id: str,
711
+ prompt: PromptType,
712
+ params: Union[SamplingParams, PoolingParams],
713
+ arrival_time: Optional[float] = None,
714
+ lora_request: Optional[LoRARequest] = None,
715
+ trace_headers: Optional[Mapping[str, str]] = None,
716
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
717
+ priority: int = 0,
718
+ ) -> None:
719
+ ...
720
+
721
+ @deprecate_kwargs(
722
+ "inputs",
723
+ additional_message="Please use the 'prompt' parameter instead.",
724
+ )
725
+ def add_request(
726
+ self,
727
+ request_id: str,
728
+ prompt: Optional[PromptType] = None,
729
+ params: Optional[Union[SamplingParams, PoolingParams]] = None,
730
+ arrival_time: Optional[float] = None,
731
+ lora_request: Optional[LoRARequest] = None,
732
+ trace_headers: Optional[Mapping[str, str]] = None,
733
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
734
+ priority: int = 0,
735
+ *,
736
+ inputs: Optional[PromptType] = None, # DEPRECATED
737
) -> None:
738
"""Add a request to the engine's request pool.
739
@@ -708,8 +743,7 @@ class LLMEngine:
743
744
Args:
745
request_id: The unique ID of the request.
711
- inputs: The inputs to the LLM. See
712
- :class:`~vllm.inputs.PromptInputs`
746
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
747
for more details about the format of each input.
748
params: Parameters for sampling or pooling.
749
:class:`~vllm.SamplingParams` for text generation.
@@ -744,6 +778,10 @@ class LLMEngine:
778
>>> # continue the request processing
779
>>> ...
780
"""
781
+ if inputs is not None:
782
+ prompt = inputs
783
+ assert prompt is not None and params is not None
784
+
785
if lora_request is not None and not self.lora_config:
786
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
787
"not enabled!")
@@ -756,7 +794,7 @@ class LLMEngine:
794
arrival_time = time.time()
795
796
preprocessed_inputs = self.input_preprocessor.preprocess(
759
- inputs,
797
+ prompt,
798
request_id=request_id,
799
lora_request=lora_request,
800
prompt_adapter_request=prompt_adapter_request,
vllm/engine/multiprocessing/__init__.py
+58
-3
@@ -1,13 +1,14 @@
1
from dataclasses import dataclass
2
from enum import Enum
3
-from typing import List, Mapping, Optional, Union
3
+from typing import List, Mapping, Optional, Union, overload
4
5
from vllm import PoolingParams
6
-from vllm.inputs import PromptInputs
6
+from vllm.inputs import PromptType
7
from vllm.lora.request import LoRARequest
8
from vllm.outputs import RequestOutput
9
from vllm.prompt_adapter.request import PromptAdapterRequest
10
from vllm.sampling_params import SamplingParams
11
+from vllm.utils import deprecate_kwargs
12
13
VLLM_RPC_SUCCESS_STR = "SUCCESS"
14
@@ -23,13 +24,67 @@ class MQEngineDeadError(RuntimeError):
24
25
@dataclass
26
class RPCProcessRequest:
26
- inputs: PromptInputs
27
+ prompt: PromptType
28
params: Union[SamplingParams, PoolingParams]
29
request_id: str
30
lora_request: Optional[LoRARequest] = None
31
trace_headers: Optional[Mapping[str, str]] = None
32
prompt_adapter_request: Optional[PromptAdapterRequest] = None
33
34
+ @overload # DEPRECATED
35
+ def __init__(
36
+ self,
37
+ *,
38
+ inputs: PromptType,
39
+ params: Union[SamplingParams, PoolingParams],
40
+ request_id: str,
41
+ lora_request: Optional[LoRARequest] = None,
42
+ trace_headers: Optional[Mapping[str, str]] = None,
43
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
44
+ ) -> None:
45
+ ...
46
+
47
+ @overload
48
+ def __init__(
49
+ self,
50
+ prompt: PromptType,
51
+ params: Union[SamplingParams, PoolingParams],
52
+ request_id: str,
53
+ lora_request: Optional[LoRARequest] = None,
54
+ trace_headers: Optional[Mapping[str, str]] = None,
55
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
56
+ ) -> None:
57
+ ...
58
+
59
+ @deprecate_kwargs(
60
+ "inputs",
61
+ additional_message="Please use the 'prompt' parameter instead.",
62
+ )
63
+ def __init__(
64
+ self,
65
+ prompt: Optional[PromptType] = None,
66
+ params: Optional[Union[SamplingParams, PoolingParams]] = None,
67
+ request_id: Optional[str] = None,
68
+ lora_request: Optional[LoRARequest] = None,
69
+ trace_headers: Optional[Mapping[str, str]] = None,
70
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
71
+ *,
72
+ inputs: Optional[PromptType] = None, # DEPRECATED
73
+ ) -> None:
74
+ if inputs is not None:
75
+ prompt = inputs
76
+ assert (prompt is not None and params is not None
77
+ and request_id is not None)
78
+
79
+ super().__init__()
80
+
81
+ self.prompt = prompt
82
+ self.params = params
83
+ self.request_id = request_id
84
+ self.lora_request = lora_request
85
+ self.trace_headers = trace_headers
86
+ self.prompt_adapter_request = prompt_adapter_request
87
+
88
89
@dataclass
90
class RPCError:
vllm/engine/multiprocessing/client.py
+82
-13
@@ -3,7 +3,7 @@ import copy
3
import pickle
4
from contextlib import contextmanager, suppress
5
from typing import (Any, AsyncGenerator, Dict, Iterator, Mapping, Optional,
6
- Union)
6
+ Union, overload)
7
8
import cloudpickle
9
import zmq
@@ -25,13 +25,14 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25
RPCUProfileRequest)
26
# yapf: enable
27
from vllm.envs import VLLM_RPC_TIMEOUT
28
-from vllm.inputs import PromptInputs
28
+from vllm.inputs import PromptType
29
from vllm.logger import init_logger
30
from vllm.lora.request import LoRARequest
31
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
32
from vllm.prompt_adapter.request import PromptAdapterRequest
33
from vllm.sampling_params import SamplingParams
34
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
35
+from vllm.utils import deprecate_kwargs
36
37
logger = init_logger(__name__)
38
@@ -367,14 +368,45 @@ class MQLLMEngineClient:
368
def dead_error(self) -> BaseException:
369
return ENGINE_DEAD_ERROR(self._errored_with)
370
371
+ @overload # DEPRECATED
372
def generate(
373
self,
372
- inputs: PromptInputs,
374
+ *,
375
+ inputs: PromptType,
376
sampling_params: SamplingParams,
377
request_id: str,
378
lora_request: Optional[LoRARequest] = None,
379
trace_headers: Optional[Mapping[str, str]] = None,
377
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
380
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
381
+ ) -> AsyncGenerator[RequestOutput, None]:
382
+ ...
383
+
384
+ @overload
385
+ def generate(
386
+ self,
387
+ prompt: PromptType,
388
+ sampling_params: SamplingParams,
389
+ request_id: str,
390
+ lora_request: Optional[LoRARequest] = None,
391
+ trace_headers: Optional[Mapping[str, str]] = None,
392
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
393
+ ) -> AsyncGenerator[RequestOutput, None]:
394
+ ...
395
+
396
+ @deprecate_kwargs(
397
+ "inputs",
398
+ additional_message="Please use the 'prompt' parameter instead.",
399
+ )
400
+ def generate(
401
+ self,
402
+ prompt: Optional[PromptType] = None,
403
+ sampling_params: Optional[SamplingParams] = None,
404
+ request_id: Optional[str] = None,
405
+ lora_request: Optional[LoRARequest] = None,
406
+ trace_headers: Optional[Mapping[str, str]] = None,
407
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
408
+ *,
409
+ inputs: Optional[PromptType] = None # DEPRECATED
410
) -> AsyncGenerator[RequestOutput, None]:
411
"""Generate outputs for a request.
412
@@ -383,8 +415,7 @@ class MQLLMEngineClient:
415
from the LLMEngine to the caller.
416
417
Args:
386
- inputs: The inputs to the LLM. See
387
- :class:`~vllm.inputs.PromptInputs`
418
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
419
for more details about the format of each input.
420
sampling_params: The sampling parameters of the request.
421
request_id: The unique id of the request.
@@ -393,17 +424,51 @@ class MQLLMEngineClient:
424
prompt_adapter_request: Prompt Adapter request to use
425
for generation, if any.
426
"""
396
- return self._process_request(inputs, sampling_params, request_id,
427
+ if inputs is not None:
428
+ prompt = inputs
429
+ assert (prompt is not None and sampling_params is not None
430
+ and request_id is not None)
431
+
432
+ return self._process_request(prompt, sampling_params, request_id,
433
lora_request, trace_headers,
434
prompt_adapter_request)
435
436
+ @overload # DEPRECATED
437
def encode(
438
self,
402
- inputs: PromptInputs,
439
+ *,
440
+ inputs: PromptType,
441
pooling_params: PoolingParams,
442
request_id: str,
443
lora_request: Optional[LoRARequest] = None,
444
trace_headers: Optional[Mapping[str, str]] = None,
445
+ ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
446
+ ...
447
+
448
+ @overload
449
+ def encode(
450
+ self,
451
+ prompt: PromptType,
452
+ pooling_params: PoolingParams,
453
+ request_id: str,
454
+ lora_request: Optional[LoRARequest] = None,
455
+ trace_headers: Optional[Mapping[str, str]] = None,
456
+ ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
457
+ ...
458
+
459
+ @deprecate_kwargs(
460
+ "inputs",
461
+ additional_message="Please use the 'prompt' parameter instead.",
462
+ )
463
+ def encode(
464
+ self,
465
+ prompt: Optional[PromptType] = None,
466
+ pooling_params: Optional[PoolingParams] = None,
467
+ request_id: Optional[str] = None,
468
+ lora_request: Optional[LoRARequest] = None,
469
+ trace_headers: Optional[Mapping[str, str]] = None,
470
+ *,
471
+ inputs: Optional[PromptType] = None # DEPRECATED
472
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
473
"""Generate outputs for a request from an embedding model.
474
@@ -412,8 +477,7 @@ class MQLLMEngineClient:
477
from the LLMEngine to the caller.
478
479
Args:
415
- inputs: The inputs to the LLM. See
416
- :class:`~vllm.inputs.PromptInputs`
480
+ prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
481
for more details about the format of each input.
482
pooling_params: The pooling parameters of the request.
483
request_id: The unique id of the request.
@@ -424,12 +488,17 @@ class MQLLMEngineClient:
488
The output `EmbeddingRequestOutput` objects from the LLMEngine
489
for the request.
490
"""
427
- return self._process_request(inputs, pooling_params, request_id,
491
+ if inputs is not None:
492
+ prompt = inputs
493
+ assert (prompt is not None and pooling_params is not None
494
+ and request_id is not None)
495
+
496
+ return self._process_request(prompt, pooling_params, request_id,
497
lora_request, trace_headers)
498
499
async def _process_request(
500
self,
432
- inputs: PromptInputs,
501
+ prompt: PromptType,
502
params: Union[SamplingParams, PoolingParams],
503
request_id: str,
504
lora_request: Optional[LoRARequest] = None,
@@ -462,7 +531,7 @@ class MQLLMEngineClient:
531
532
request_bytes = pickle.dumps(
533
RPCProcessRequest(
465
- inputs=inputs,
534
+ prompt=prompt,
535
params=params,
536
request_id=request_id,
537
lora_request=lora_request,
vllm/engine/multiprocessing/engine.py
+1
-1
@@ -278,7 +278,7 @@ class MQLLMEngine:
278
try:
279
self.engine.add_request(
280
request_id=request_id,
281
- inputs=request.inputs,
281
+ prompt=request.prompt,
282
params=request.params,
283
lora_request=request.lora_request,
284
trace_headers=request.trace_headers,
vllm/engine/protocol.py
+4
-4
@@ -3,7 +3,7 @@ from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
3
4
from vllm.config import DecodingConfig, ModelConfig
5
from vllm.core.scheduler import SchedulerOutputs
6
-from vllm.inputs.data import PromptInputs
6
+from vllm.inputs.data import PromptType
7
from vllm.lora.request import LoRARequest
8
from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -35,19 +35,19 @@ class EngineClient(Protocol):
35
36
def generate(
37
self,
38
- inputs: PromptInputs,
38
+ prompt: PromptType,
39
sampling_params: SamplingParams,
40
request_id: str,
41
lora_request: Optional[LoRARequest] = None,
42
trace_headers: Optional[Mapping[str, str]] = None,
43
prompt_adapter_request: Optional[PromptAdapterRequest] = None
44
) -> AsyncGenerator[RequestOutput, None]:
45
- """Generates outputs for a request"""
45
+ """Generate outputs for a request."""
46
...
47
48
def encode(
49
self,
50
- inputs: PromptInputs,
50
+ prompt: PromptType,
51
pooling_params: PoolingParams,
52
request_id: str,
53
lora_request: Optional[LoRARequest] = None,
vllm/entrypoints/llm.py
+35
-33
@@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
12
apply_hf_chat_template,
13
apply_mistral_chat_template,
14
parse_chat_messages)
15
-from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
15
+from vllm.inputs import PromptType, TextPrompt, TokensPrompt
16
from vllm.inputs.parse import parse_and_batch_prompt
17
from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
@@ -293,8 +293,8 @@ class LLM:
293
@overload
294
def generate(
295
self,
296
- inputs: Union[PromptInputs, Sequence[PromptInputs]],
297
- /, # We may enable `inputs` keyword after removing the old API
296
+ prompts: Union[PromptType, Sequence[PromptType]],
297
+ /,
298
*,
299
sampling_params: Optional[Union[SamplingParams,
300
Sequence[SamplingParams]]] = None,
@@ -304,14 +304,13 @@ class LLM:
304
...
305
306
@deprecate_kwargs(
307
- "prompts",
307
"prompt_token_ids",
308
is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
310
- additional_message="Please use the 'inputs' parameter instead.",
309
+ additional_message="Please use the 'prompts' parameter instead.",
310
)
311
def generate(
312
self,
314
- prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
313
+ prompts: Union[Union[PromptType, Sequence[PromptType]],
314
Optional[Union[str, List[str]]]] = None,
315
sampling_params: Optional[Union[SamplingParams,
316
Sequence[SamplingParams]]] = None,
@@ -330,7 +329,9 @@ class LLM:
329
into a single list and pass it to this method.
330
331
Args:
333
- inputs: A list of inputs to generate completions for.
332
+ prompts: The prompts to the LLM. You may pass a sequence of prompts
333
+ for batch inference. See :class:`~vllm.inputs.PromptType`
334
+ for more details about the format of each prompts.
335
sampling_params: The sampling parameters for text generation. If
336
None, we use the default sampling parameters.
337
When it is a single value, it is applied to every prompt.
@@ -358,12 +359,13 @@ class LLM:
359
"models (XForCausalLM, XForConditionalGeneration).")
360
361
if prompt_token_ids is not None:
361
- inputs = self._convert_v1_inputs(
362
+ parsed_prompts = self._convert_v1_inputs(
363
prompts=cast(Optional[Union[str, List[str]]], prompts),
364
prompt_token_ids=prompt_token_ids,
365
)
366
else:
366
- inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
367
+ parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
368
+ prompts)
369
370
if isinstance(guided_options_request, dict):
371
if len(guided_options_request) > 1:
@@ -378,7 +380,7 @@ class LLM:
380
sampling_params = SamplingParams()
381
382
self._validate_and_add_requests(
381
- inputs=inputs,
383
+ prompts=parsed_prompts,
384
params=sampling_params,
385
lora_request=lora_request,
386
prompt_adapter_request=prompt_adapter_request,
@@ -648,8 +650,8 @@ class LLM:
650
@overload
651
def encode(
652
self,
651
- inputs: Union[PromptInputs, Sequence[PromptInputs]],
652
- /, # We may enable `inputs` keyword after removing the old API
653
+ prompts: Union[PromptType, Sequence[PromptType]],
654
+ /,
655
*,
656
pooling_params: Optional[Union[PoolingParams,
657
Sequence[PoolingParams]]] = None,
@@ -659,14 +661,13 @@ class LLM:
661
...
662
663
@deprecate_kwargs(
662
- "prompts",
664
"prompt_token_ids",
665
is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
665
- additional_message="Please use the 'inputs' parameter instead.",
666
+ additional_message="Please use the 'prompts' parameter instead.",
667
)
668
def encode(
669
self,
669
- prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
670
+ prompts: Union[Union[PromptType, Sequence[PromptType]],
671
Optional[Union[str, List[str]]]] = None,
672
pooling_params: Optional[Union[PoolingParams,
673
Sequence[PoolingParams]]] = None,
@@ -682,9 +683,9 @@ class LLM:
683
into a single list and pass it to this method.
684
685
Args:
685
- inputs: The inputs to the LLM. You may pass a sequence of inputs for
686
- batch inference. See :class:`~vllm.inputs.PromptInputs`
687
- for more details about the format of each input.
686
+ prompts: The prompts to the LLM. You may pass a sequence of prompts
687
+ for batch inference. See :class:`~vllm.inputs.PromptType`
688
+ for more details about the format of each prompts.
689
pooling_params: The pooling parameters for pooling. If None, we
690
use the default pooling parameters.
691
use_tqdm: Whether to use tqdm to display the progress bar.
@@ -707,19 +708,20 @@ class LLM:
708
)
709
710
if prompt_token_ids is not None:
710
- inputs = self._convert_v1_inputs(
711
+ parsed_prompts = self._convert_v1_inputs(
712
prompts=cast(Optional[Union[str, List[str]]], prompts),
713
prompt_token_ids=prompt_token_ids,
714
)
715
else:
715
- inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
716
+ parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
717
+ prompts)
718
719
if pooling_params is None:
720
# Use default pooling params.
721
pooling_params = PoolingParams()
722
723
self._validate_and_add_requests(
722
- inputs=inputs,
724
+ prompts=parsed_prompts,
725
params=pooling_params,
726
lora_request=lora_request,
727
prompt_adapter_request=prompt_adapter_request,
@@ -763,9 +765,9 @@ class LLM:
765
raise ValueError("Either prompts or prompt_token_ids must be "
766
"provided.")
767
766
- inputs: List[PromptInputs] = []
768
+ parsed_prompts: List[PromptType] = []
769
for i in range(num_requests):
768
- item: PromptInputs
770
+ item: PromptType
771
772
if prompts is not None:
773
item = TextPrompt(prompt=prompts[i])
@@ -774,13 +776,13 @@ class LLM:
776
else:
777
raise AssertionError
778
777
- inputs.append(item)
779
+ parsed_prompts.append(item)
780
779
- return inputs
781
+ return parsed_prompts
782
783
def _validate_and_add_requests(
784
self,
783
- inputs: Union[PromptInputs, Sequence[PromptInputs]],
785
+ prompts: Union[PromptType, Sequence[PromptType]],
786
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
787
Sequence[PoolingParams]],
788
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
@@ -788,11 +790,11 @@ class LLM:
790
guided_options: Optional[GuidedDecodingRequest] = None,
791
priority: Optional[List[int]] = None,
792
) -> None:
791
- if isinstance(inputs, (str, dict)):
793
+ if isinstance(prompts, (str, dict)):
794
# Convert a single prompt to a list.
793
- inputs = [inputs]
795
+ prompts = [prompts]
796
795
- num_requests = len(inputs)
797
+ num_requests = len(prompts)
798
if isinstance(params, list) and len(params) != num_requests:
799
raise ValueError("The lengths of prompts and params "
800
"must be the same.")
@@ -809,9 +811,9 @@ class LLM:
811
sp.output_kind = RequestOutputKind.FINAL_ONLY
812
813
# Add requests to the engine.
812
- for i, request_inputs in enumerate(inputs):
814
+ for i, prompt in enumerate(prompts):
815
self._add_request(
814
- request_inputs,
816
+ prompt,
817
params[i] if isinstance(params, Sequence) else params,
818
lora_request=lora_request[i] if isinstance(
819
lora_request, Sequence) else lora_request,
@@ -821,7 +823,7 @@ class LLM:
823
824
def _add_request(
825
self,
824
- inputs: PromptInputs,
826
+ prompt: PromptType,
827
params: Union[SamplingParams, PoolingParams],
828
lora_request: Optional[LoRARequest] = None,
829
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -830,7 +832,7 @@ class LLM:
832
request_id = str(next(self.request_counter))
833
self.llm_engine.add_request(
834
request_id,
833
- inputs,
835
+ prompt,
836
params,
837
lora_request=lora_request,
838
prompt_adapter_request=prompt_adapter_request,
vllm/inputs/__init__.py
+17
-3
@@ -1,5 +1,5 @@
1
from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2
- LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
2
+ LLMInputs, PromptType, SingletonPrompt, TextPrompt,
3
TokensPrompt, build_explicit_enc_dec_prompt,
4
to_enc_dec_tuple_list, zip_enc_dec_prompts)
5
from .registry import InputContext, InputRegistry
@@ -16,8 +16,8 @@ See also:
16
__all__ = [
17
"TextPrompt",
18
"TokensPrompt",
19
- "PromptInputs",
20
- "SingletonPromptInputs",
19
+ "PromptType",
20
+ "SingletonPrompt",
21
"ExplicitEncoderDecoderPrompt",
22
"LLMInputs",
23
"EncoderDecoderLLMInputs",
@@ -28,3 +28,17 @@ __all__ = [
28
"InputContext",
29
"InputRegistry",
30
]
31
+
32
+
33
+def __getattr__(name: str):
34
+ if name == "PromptInput":
35
+ import warnings
36
+
37
+ msg = ("PromptInput has been renamed to PromptType. "
38
+ "The original name will be removed in an upcoming version.")
39
+
40
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
41
+
42
+ return PromptType
43
+
44
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/data.py
+31
-22
@@ -33,7 +33,7 @@ class TokensPrompt(TypedDict):
33
"""
34
35
36
-SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
36
+SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
37
"""
38
Set of possible schemas for a single LLM input:
39
@@ -46,7 +46,7 @@ which may be utilized for encoder/decoder models when
46
the user desires to express both the encoder & decoder
47
prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
49
-A prompt of type :class:`SingletonPromptInputs` may be employed
49
+A prompt of type :class:`SingletonPrompt` may be employed
50
as (1) input to a decoder-only model, (2) input to
51
the encoder of an encoder/decoder model, in the scenario
52
where the decoder-prompt is not specified explicitly, or
@@ -55,33 +55,32 @@ more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55
"""
56
57
_T1_co = TypeVar("_T1_co",
58
- bound=SingletonPromptInputs,
59
- default=SingletonPromptInputs,
58
+ bound=SingletonPrompt,
59
+ default=SingletonPrompt,
60
covariant=True)
61
_T2_co = TypeVar("_T2_co",
62
- bound=SingletonPromptInputs,
63
- default=SingletonPromptInputs,
62
+ bound=SingletonPrompt,
63
+ default=SingletonPrompt,
64
covariant=True)
65
66
67
# TODO: Make fields ReadOnly once mypy supports it
68
class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
69
- """Represents an encoder/decoder model input prompt,
70
- comprising an explicit encoder prompt and a
71
- decoder prompt.
69
+ """
70
+ Represents an encoder/decoder model input prompt,
71
+ comprising an explicit encoder prompt and a decoder prompt.
72
73
- The encoder and decoder prompts, respectively,
74
- may formatted according to any of the
75
- :class:`SingletonPromptInputs` schemas, and are not
76
- required to have the same schema.
73
+ The encoder and decoder prompts, respectively, may be formatted
74
+ according to any of the :class:`SingletonPrompt` schemas,
75
+ and are not required to have the same schema.
76
77
Only the encoder prompt may have multi-modal data.
78
79
Note that an :class:`ExplicitEncoderDecoderPrompt` may not
80
be used as an input to a decoder-only model,
82
- and that the `encoder_prompt` and `decoder_prompt`
81
+ and that the :code:`encoder_prompt` and :code:`decoder_prompt`
82
fields of this data structure themselves must be
84
- :class:`SingletonPromptInputs` instances.
83
+ :class:`SingletonPrompt` instances.
84
"""
85
86
encoder_prompt: _T1_co
@@ -89,7 +88,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
88
decoder_prompt: Optional[_T2_co]
89
90
92
-PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
91
+PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
92
"""
93
Set of possible schemas for an LLM input, including
94
both decoder-only and encoder/decoder input types:
@@ -146,12 +145,8 @@ class EncoderDecoderLLMInputs(LLMInputs):
145
"""
146
147
149
-_T1 = TypeVar("_T1",
150
- bound=SingletonPromptInputs,
151
- default=SingletonPromptInputs)
152
-_T2 = TypeVar("_T2",
153
- bound=SingletonPromptInputs,
154
- default=SingletonPromptInputs)
148
+_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
149
+_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
150
151
152
def build_explicit_enc_dec_prompt(
@@ -182,3 +177,17 @@ def to_enc_dec_tuple_list(
177
return [(enc_dec_prompt["encoder_prompt"],
178
enc_dec_prompt["decoder_prompt"])
179
for enc_dec_prompt in enc_dec_prompts]
180
+
181
+
182
+def __getattr__(name: str):
183
+ if name == "PromptInput":
184
+ import warnings
185
+
186
+ msg = ("PromptInput has been renamed to PromptType. "
187
+ "The original name will be removed in an upcoming version.")
188
+
189
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
190
+
191
+ return PromptType
192
+
193
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/parse.py
+11
-11
@@ -5,7 +5,7 @@ from typing_extensions import TypeIs
5
from vllm.utils import is_list_of
6
7
from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8
- LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
8
+ LLMInputs, PromptType, SingletonPrompt, TextPrompt,
9
TokensPrompt)
10
11
@@ -81,23 +81,23 @@ class ParsedTokensPrompt(TypedDict):
81
82
83
def parse_singleton_prompt(
84
- inputs: SingletonPromptInputs,
84
+ prompt: SingletonPrompt,
85
) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86
- if isinstance(inputs, str):
87
- return ParsedStrPrompt(type="str", content=inputs)
88
- elif isinstance(inputs, dict):
89
- if "prompt_token_ids" in inputs:
86
+ if isinstance(prompt, str):
87
+ return ParsedStrPrompt(type="str", content=prompt)
88
+ elif isinstance(prompt, dict):
89
+ if "prompt_token_ids" in prompt:
90
return ParsedTokensPrompt(type="tokens",
91
- content=inputs) # type: ignore
92
- elif "prompt" in inputs:
93
- return ParsedTextPrompt(type="text", content=inputs)
91
+ content=prompt) # type: ignore
92
+ elif "prompt" in prompt:
93
+ return ParsedTextPrompt(type="text", content=prompt)
94
95
raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98
def is_explicit_encoder_decoder_prompt(
99
- inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100
- return isinstance(inputs, dict) and "encoder_prompt" in inputs
99
+ prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100
+ return isinstance(prompt, dict) and "encoder_prompt" in prompt
101
102
103
def is_valid_encoder_decoder_llm_inputs(
vllm/inputs/preprocess.py
+43
-43
@@ -10,8 +10,8 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
10
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
from vllm.utils import print_warning_once
12
13
-from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
14
- SingletonPromptInputs)
13
+from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
14
+ SingletonPrompt)
15
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
16
17
if TYPE_CHECKING:
@@ -209,7 +209,7 @@ class InputPreprocessor:
209
210
def _extract_prompt_components(
211
self,
212
- inputs: SingletonPromptInputs,
212
+ prompt: SingletonPrompt,
213
request_id: str,
214
lora_request: Optional[LoRARequest] = None,
215
) -> PromptComponents:
@@ -219,7 +219,7 @@ class InputPreprocessor:
219
Arguments:
220
221
* request_id
222
- * inputs: single encoder or decoder input prompt
222
+ * prompt: single encoder or decoder input prompt
223
* lora_request: this is only valid for decoder prompts
224
225
Returns:
@@ -229,24 +229,24 @@ class InputPreprocessor:
229
* multi_modal_data
230
'''
231
232
- parsed = parse_singleton_prompt(inputs)
232
+ parsed = parse_singleton_prompt(prompt)
233
234
if parsed["type"] == "str":
235
- prompt = parsed["content"]
235
+ prompt_text = parsed["content"]
236
prompt_token_ids = self._tokenize_prompt(
237
- prompt,
237
+ prompt_text,
238
request_id=request_id,
239
lora_request=lora_request,
240
)
241
multi_modal_data = None
242
elif parsed["type"] == "tokens":
243
- prompt = None
243
+ prompt_text = None
244
prompt_token_ids = parsed["content"]["prompt_token_ids"]
245
multi_modal_data = parsed["content"].get("multi_modal_data")
246
elif parsed["type"] == "text":
247
- prompt = parsed["content"]["prompt"]
247
+ prompt_text = parsed["content"]["prompt"]
248
prompt_token_ids = self._tokenize_prompt(
249
- prompt,
249
+ prompt_text,
250
request_id=request_id,
251
lora_request=lora_request,
252
)
@@ -254,33 +254,33 @@ class InputPreprocessor:
254
else:
255
assert_never(parsed)
256
257
- return prompt, prompt_token_ids, multi_modal_data
257
+ return prompt_text, prompt_token_ids, multi_modal_data
258
259
async def _extract_prompt_components_async(
260
self,
261
- inputs: SingletonPromptInputs,
261
+ prompt: SingletonPrompt,
262
request_id: str,
263
lora_request: Optional[LoRARequest] = None,
264
) -> PromptComponents:
265
"""Async version of :meth:`_extract_prompt_components`."""
266
- parsed = parse_singleton_prompt(inputs)
266
+ parsed = parse_singleton_prompt(prompt)
267
268
if parsed["type"] == "str":
269
- prompt = parsed["content"]
269
+ prompt_text = parsed["content"]
270
prompt_token_ids = await self._tokenize_prompt_async(
271
- prompt,
271
+ prompt_text,
272
request_id=request_id,
273
lora_request=lora_request,
274
)
275
multi_modal_data = None
276
elif parsed["type"] == "tokens":
277
- prompt = None
277
+ prompt_text = None
278
prompt_token_ids = parsed["content"]["prompt_token_ids"]
279
multi_modal_data = parsed["content"].get("multi_modal_data")
280
elif parsed["type"] == "text":
281
- prompt = parsed["content"]["prompt"]
281
+ prompt_text = parsed["content"]["prompt"]
282
prompt_token_ids = await self._tokenize_prompt_async(
283
- prompt,
283
+ prompt_text,
284
request_id=request_id,
285
lora_request=lora_request,
286
)
@@ -288,7 +288,7 @@ class InputPreprocessor:
288
else:
289
assert_never(parsed)
290
291
- return prompt, prompt_token_ids, multi_modal_data
291
+ return prompt_text, prompt_token_ids, multi_modal_data
292
293
def _build_enc_dec_llm_inputs(
294
self,
@@ -321,7 +321,7 @@ class InputPreprocessor:
321
322
def _process_encoder_decoder_prompt(
323
self,
324
- inputs: PromptInputs,
324
+ prompt: PromptType,
325
request_id: str,
326
) -> EncoderDecoderLLMInputs:
327
'''
@@ -349,7 +349,7 @@ class InputPreprocessor:
349
350
Arguments:
351
352
- * inputs: an input prompt
352
+ * prompt: an input prompt
353
* request_id
354
355
Returns:
@@ -360,13 +360,13 @@ class InputPreprocessor:
360
encoder_comps: PromptComponents
361
decoder_comps: DecoderPromptComponents
362
363
- if is_explicit_encoder_decoder_prompt(inputs):
363
+ if is_explicit_encoder_decoder_prompt(prompt):
364
encoder_comps = self._extract_prompt_components(
365
- inputs["encoder_prompt"],
365
+ prompt["encoder_prompt"],
366
request_id=request_id,
367
)
368
369
- if (decoder_input := inputs["decoder_prompt"]) is None:
369
+ if (decoder_input := prompt["decoder_prompt"]) is None:
370
decoder_comps = None, None, None
371
else:
372
decoder_comps = self._extract_prompt_components(
@@ -375,7 +375,7 @@ class InputPreprocessor:
375
)
376
else:
377
encoder_comps = self._extract_prompt_components(
378
- inputs,
378
+ prompt,
379
request_id=request_id,
380
)
381
@@ -385,20 +385,20 @@ class InputPreprocessor:
385
386
async def _process_encoder_decoder_prompt_async(
387
self,
388
- inputs: PromptInputs,
388
+ prompt: PromptType,
389
request_id: str,
390
) -> EncoderDecoderLLMInputs:
391
"""Async version of :meth:`_process_encoder_decoder_prompt`."""
392
encoder_comps: PromptComponents
393
decoder_comps: DecoderPromptComponents
394
395
- if is_explicit_encoder_decoder_prompt(inputs):
395
+ if is_explicit_encoder_decoder_prompt(prompt):
396
encoder_task = self._extract_prompt_components_async(
397
- inputs["encoder_prompt"],
397
+ prompt["encoder_prompt"],
398
request_id=request_id,
399
)
400
401
- if (decoder_input := inputs["decoder_prompt"]) is None:
401
+ if (decoder_input := prompt["decoder_prompt"]) is None:
402
encoder_comps = await encoder_task
403
decoder_comps = None, None, None
404
else:
@@ -411,7 +411,7 @@ class InputPreprocessor:
411
encoder_task, decoder_task)
412
else:
413
encoder_comps = await self._extract_prompt_components_async(
414
- inputs,
414
+ prompt,
415
request_id=request_id,
416
)
417
@@ -435,7 +435,7 @@ class InputPreprocessor:
435
436
def _process_decoder_only_prompt(
437
self,
438
- inputs: SingletonPromptInputs,
438
+ prompt: SingletonPrompt,
439
request_id: str,
440
lora_request: Optional[LoRARequest] = None,
441
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -446,7 +446,7 @@ class InputPreprocessor:
446
447
Arguments:
448
449
- * inputs: input prompt
449
+ * prompt: input prompt
450
* request_id
451
* lora_request
452
* prompt_adapter_request
@@ -457,7 +457,7 @@ class InputPreprocessor:
457
'''
458
459
prompt_comps = self._extract_prompt_components(
460
- inputs,
460
+ prompt,
461
request_id=request_id,
462
lora_request=lora_request,
463
)
@@ -469,14 +469,14 @@ class InputPreprocessor:
469
470
async def _process_decoder_only_prompt_async(
471
self,
472
- inputs: SingletonPromptInputs,
472
+ prompt: SingletonPrompt,
473
request_id: str,
474
lora_request: Optional[LoRARequest] = None,
475
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
476
) -> LLMInputs:
477
"""Async version of :meth:`_process_decoder_only_prompt`."""
478
prompt_comps = await self._extract_prompt_components_async(
479
- inputs,
479
+ prompt,
480
request_id=request_id,
481
lora_request=lora_request,
482
)
@@ -488,7 +488,7 @@ class InputPreprocessor:
488
489
def preprocess(
490
self,
491
- inputs: PromptInputs,
491
+ prompt: PromptType,
492
request_id: str,
493
lora_request: Optional[LoRARequest] = None,
494
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -498,17 +498,17 @@ class InputPreprocessor:
498
# Encoder-decoder model requires special mapping of
499
# input prompts to encoder & decoder
500
return self._process_encoder_decoder_prompt(
501
- inputs,
501
+ prompt,
502
request_id=request_id,
503
)
504
505
- if is_explicit_encoder_decoder_prompt(inputs):
505
+ if is_explicit_encoder_decoder_prompt(prompt):
506
raise ValueError("Cannot pass encoder-decoder prompt "
507
"to decoder-only models")
508
509
# Decoder-only operation
510
return self._process_decoder_only_prompt(
511
- inputs,
511
+ prompt,
512
request_id=request_id,
513
lora_request=lora_request,
514
prompt_adapter_request=prompt_adapter_request,
@@ -516,7 +516,7 @@ class InputPreprocessor:
516
517
async def preprocess_async(
518
self,
519
- inputs: PromptInputs,
519
+ prompt: PromptType,
520
request_id: str,
521
lora_request: Optional[LoRARequest] = None,
522
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -526,17 +526,17 @@ class InputPreprocessor:
526
# Encoder-decoder model requires special mapping of
527
# input prompts to encoder & decoder
528
return await self._process_encoder_decoder_prompt_async(
529
- inputs,
529
+ prompt,
530
request_id=request_id,
531
)
532
533
- if is_explicit_encoder_decoder_prompt(inputs):
533
+ if is_explicit_encoder_decoder_prompt(prompt):
534
raise ValueError("Cannot pass encoder-decoder prompt "
535
"to decoder-only models")
536
537
# Decoder-only operation
538
return await self._process_decoder_only_prompt_async(
539
- inputs,
539
+ prompt,
540
request_id=request_id,
541
lora_request=lora_request,
542
prompt_adapter_request=prompt_adapter_request,