A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
Revert "rename PromptInputs and inputs with backward compatibility (#8760) (#8810)
Simon Mo committed
Sep 25, 2024 at 10:36 UTC
4f1ba0844b83b4e7d0ff1672b7ba502ce8732f95
21 files changed
+245
-438
benchmarks/benchmark_latency.py
+4
-4
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12
from vllm import LLM, SamplingParams
13
from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14
-from vllm.inputs import PromptType
14
+from vllm.inputs import PromptInputs
15
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61
dummy_prompt_token_ids = np.random.randint(10000,
62
size=(args.batch_size,
63
args.input_len))
64
- dummy_prompts: List[PromptType] = [{
64
+ dummy_inputs: List[PromptInputs] = [{
65
"prompt_token_ids": batch
66
} for batch in dummy_prompt_token_ids.tolist()]
67
@@ -74,13 +74,13 @@ def main(args: argparse.Namespace):
74
],
75
on_trace_ready=torch.profiler.tensorboard_trace_handler(
76
str(profile_dir))) as p:
77
- llm.generate(dummy_prompts,
77
+ llm.generate(dummy_inputs,
78
sampling_params=sampling_params,
79
use_tqdm=False)
80
print(p.key_averages())
81
else:
82
start_time = time.perf_counter()
83
- llm.generate(dummy_prompts,
83
+ llm.generate(dummy_inputs,
84
sampling_params=sampling_params,
85
use_tqdm=False)
86
end_time = time.perf_counter()
docs/source/dev/multimodal/multimodal_index.rst
+1
-1
@@ -8,7 +8,7 @@ Multi-Modality
8
vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10
Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11
-via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
11
+via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
12
13
Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14
by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1
-1
@@ -1,7 +1,7 @@
1
LLM Inputs
2
==========
3
4
-.. autodata:: vllm.inputs.PromptType
4
+.. autodata:: vllm.inputs.PromptInputs
5
6
.. autoclass:: vllm.inputs.TextPrompt
7
:show-inheritance:
docs/source/models/vlm.rst
+1
-1
@@ -27,7 +27,7 @@ The :class:`~vllm.LLM` class can be instantiated in much the same way as languag
27
We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28
the above snippet. Specifically, ``image_feature_size`` can no longer be specified as we now calculate that internally for each model.
29
30
-To pass an image to the model, note the following in :class:`vllm.inputs.PromptType`:
30
+To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
31
32
* ``prompt``: The prompt should follow the format that is documented on HuggingFace.
33
* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/async_engine/test_async_llm_engine.py
+3
-5
@@ -86,19 +86,17 @@ class MockAsyncLLMEngine(AsyncLLMEngine):
86
87
@pytest.mark.asyncio
88
async def test_new_requests_event():
89
- params = SamplingParams()
90
-
89
engine = MockAsyncLLMEngine()
90
engine.start_background_loop()
91
await asyncio.sleep(0.01)
92
assert engine.engine.step_calls == 0
93
96
- await engine.add_request("1", "", params)
94
+ await engine.add_request("1", "", None)
95
await asyncio.sleep(0.01)
96
assert engine.engine.add_request_calls == 1
97
assert engine.engine.step_calls == 1
98
101
- await engine.add_request("2", "", params)
99
+ await engine.add_request("2", "", None)
100
engine.engine.generate("2")
101
await asyncio.sleep(0)
102
await asyncio.sleep(0)
@@ -113,7 +111,7 @@ async def test_new_requests_event():
111
await asyncio.sleep(0.001)
112
assert engine.engine.step_calls == old_step_calls
113
116
- await engine.add_request("3", "", params)
114
+ await engine.add_request("3", "", None)
115
await asyncio.sleep(0.01)
116
assert engine.engine.add_request_calls == 3
117
assert engine.engine.step_calls == old_step_calls + 1
tests/entrypoints/llm/test_encode.py
+34
@@ -49,6 +49,21 @@ def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
49
assert [o.outputs for o in o1] == [o.outputs for o in o2]
50
51
52
+@pytest.mark.skip_global_cleanup
53
+@pytest.mark.parametrize('prompt', PROMPTS)
54
+def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
55
+ pooling_params = PoolingParams()
56
+
57
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
58
+ v1_output = llm.encode(prompts=prompt, pooling_params=pooling_params)
59
+
60
+ v2_output = llm.encode(prompt, pooling_params=pooling_params)
61
+ assert_outputs_equal(v1_output, v2_output)
62
+
63
+ v2_output = llm.encode({"prompt": prompt}, pooling_params=pooling_params)
64
+ assert_outputs_equal(v1_output, v2_output)
65
+
66
+
67
@pytest.mark.skip_global_cleanup
68
@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
69
def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -64,6 +79,25 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
79
assert_outputs_equal(v1_output, v2_output)
80
81
82
+@pytest.mark.skip_global_cleanup
83
+def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84
+ pooling_params = PoolingParams()
85
+
86
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
87
+ v1_output = llm.encode(prompts=PROMPTS, pooling_params=pooling_params)
88
+
89
+ v2_output = llm.encode(PROMPTS, pooling_params=pooling_params)
90
+ assert_outputs_equal(v1_output, v2_output)
91
+
92
+ v2_output = llm.encode(
93
+ [{
94
+ "prompt": p
95
+ } for p in PROMPTS],
96
+ pooling_params=pooling_params,
97
+ )
98
+ assert_outputs_equal(v1_output, v2_output)
99
+
100
+
101
@pytest.mark.skip_global_cleanup
102
def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
103
pooling_params = PoolingParams()
tests/entrypoints/llm/test_generate.py
+37
@@ -47,6 +47,23 @@ def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
47
assert [o.outputs for o in o1] == [o.outputs for o in o2]
48
49
50
+@pytest.mark.skip_global_cleanup
51
+@pytest.mark.parametrize('prompt', PROMPTS)
52
+def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
53
+ sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
54
+
55
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
56
+ v1_output = llm.generate(prompts=prompt,
57
+ sampling_params=sampling_params)
58
+
59
+ v2_output = llm.generate(prompt, sampling_params=sampling_params)
60
+ assert_outputs_equal(v1_output, v2_output)
61
+
62
+ v2_output = llm.generate({"prompt": prompt},
63
+ sampling_params=sampling_params)
64
+ assert_outputs_equal(v1_output, v2_output)
65
+
66
+
67
@pytest.mark.skip_global_cleanup
68
@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
69
def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -62,6 +79,26 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
79
assert_outputs_equal(v1_output, v2_output)
80
81
82
+@pytest.mark.skip_global_cleanup
83
+def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84
+ sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
85
+
86
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
87
+ v1_output = llm.generate(prompts=PROMPTS,
88
+ sampling_params=sampling_params)
89
+
90
+ v2_output = llm.generate(PROMPTS, sampling_params=sampling_params)
91
+ assert_outputs_equal(v1_output, v2_output)
92
+
93
+ v2_output = llm.generate(
94
+ [{
95
+ "prompt": p
96
+ } for p in PROMPTS],
97
+ sampling_params=sampling_params,
98
+ )
99
+ assert_outputs_equal(v1_output, v2_output)
100
+
101
+
102
@pytest.mark.skip_global_cleanup
103
def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
104
sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
tests/mq_llm_engine/test_error_handling.py
+6
-6
@@ -61,7 +61,7 @@ async def test_evil_forward(tmp_socket):
61
62
# Throws an error in first forward pass.
63
with pytest.raises(RAISED_ERROR):
64
- async for _ in client.generate(prompt="Hello my name is",
64
+ async for _ in client.generate(inputs="Hello my name is",
65
sampling_params=SamplingParams(),
66
request_id=uuid.uuid4()):
67
pass
@@ -69,7 +69,7 @@ async def test_evil_forward(tmp_socket):
69
70
# Engine is errored, should get ENGINE_DEAD_ERROR.
71
with pytest.raises(MQEngineDeadError):
72
- async for _ in client.generate(prompt="Hello my name is",
72
+ async for _ in client.generate(inputs="Hello my name is",
73
sampling_params=SamplingParams(),
74
request_id=uuid.uuid4()):
75
pass
@@ -118,7 +118,7 @@ async def test_failed_health_check(tmp_socket):
118
119
# Generate call should throw ENGINE_DEAD_ERROR
120
with pytest.raises(MQEngineDeadError):
121
- async for _ in client.generate(prompt="Hello my name is",
121
+ async for _ in client.generate(inputs="Hello my name is",
122
sampling_params=SamplingParams(),
123
request_id=uuid.uuid4()):
124
pass
@@ -160,7 +160,7 @@ async def test_failed_abort(tmp_socket):
160
# with reference to the original KeyError("foo")
161
with pytest.raises(MQEngineDeadError) as execinfo:
162
async for _ in client.generate(
163
- prompt="Hello my name is",
163
+ inputs="Hello my name is",
164
sampling_params=SamplingParams(max_tokens=10),
165
request_id=uuid.uuid4()):
166
pass
@@ -183,7 +183,7 @@ async def test_bad_request(tmp_socket):
183
184
# Invalid request should fail, but not crash the server.
185
with pytest.raises(ValueError):
186
- async for _ in client.generate(prompt="Hello my name is",
186
+ async for _ in client.generate(inputs="Hello my name is",
187
sampling_params=SamplingParams(),
188
request_id="abcd-1",
189
lora_request=LoRARequest(
@@ -192,7 +192,7 @@ async def test_bad_request(tmp_socket):
192
pass
193
194
# This request should be okay.
195
- async for _ in client.generate(prompt="Hello my name is",
195
+ async for _ in client.generate(inputs="Hello my name is",
196
sampling_params=SamplingParams(),
197
request_id="abcd-2"):
198
pass
tests/mq_llm_engine/utils.py
+1
-1
@@ -20,7 +20,7 @@ async def generate(
20
count = 0
21
async for out in client.generate(
22
request_id=request_id,
23
- prompt="Hello my name is Robert and",
23
+ inputs="Hello my name is Robert and",
24
sampling_params=SamplingParams(max_tokens=num_tokens,
25
temperature=0)):
26
vllm/__init__.py
+2
-2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
-from vllm.inputs import PromptType, TextPrompt, TokensPrompt
8
+from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11
EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19
"__version_tuple__",
20
"LLM",
21
"ModelRegistry",
22
- "PromptType",
22
+ "PromptInputs",
23
"TextPrompt",
24
"TokensPrompt",
25
"SamplingParams",
vllm/engine/async_llm_engine.py
+18
-92
@@ -2,8 +2,8 @@ import asyncio
2
import time
3
import weakref
4
from functools import partial
5
-from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
6
- List, Mapping, Optional, Set, Tuple, Type, Union, overload)
5
+from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6
+ Mapping, Optional, Set, Tuple, Type, Union)
7
from weakref import ReferenceType
8
9
import vllm.envs as envs
@@ -17,7 +17,7 @@ from vllm.engine.metrics_types import StatLoggerBase
17
from vllm.executor.executor_base import ExecutorAsyncBase
18
from vllm.executor.gpu_executor import GPUExecutorAsync
19
from vllm.executor.ray_utils import initialize_ray_cluster
20
-from vllm.inputs import PromptType
20
+from vllm.inputs import PromptInputs
21
from vllm.logger import init_logger
22
from vllm.lora.request import LoRARequest
23
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -28,7 +28,7 @@ from vllm.sampling_params import SamplingParams
28
from vllm.sequence import ExecuteModelRequest
29
from vllm.transformers_utils.tokenizer import AnyTokenizer
30
from vllm.usage.usage_lib import UsageContext
31
-from vllm.utils import deprecate_kwargs, weak_bind
31
+from vllm.utils import weak_bind
32
33
logger = init_logger(__name__)
34
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -402,54 +402,17 @@ class _AsyncLLMEngine(LLMEngine):
402
"""Stop the remote worker execution loop."""
403
await self.model_executor.stop_remote_worker_execution_loop_async()
404
405
- @overload # DEPRECATED
405
async def add_request_async(
406
self,
407
request_id: str,
409
- *,
410
- inputs: PromptType,
408
+ inputs: PromptInputs,
409
params: Union[SamplingParams, PoolingParams],
410
arrival_time: Optional[float] = None,
411
lora_request: Optional[LoRARequest] = None,
412
trace_headers: Optional[Mapping[str, str]] = None,
413
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
416
- ) -> None:
417
- ...
418
-
419
- @overload
420
- async def add_request_async(
421
- self,
422
- request_id: str,
423
- prompt: PromptType,
424
- params: Union[SamplingParams, PoolingParams],
425
- arrival_time: Optional[float] = None,
426
- lora_request: Optional[LoRARequest] = None,
427
- trace_headers: Optional[Mapping[str, str]] = None,
428
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
429
- ) -> None:
430
- ...
431
-
432
- @deprecate_kwargs(
433
- "inputs",
434
- additional_message="Please use the 'prompt' parameter instead.",
435
- )
436
- async def add_request_async(
437
- self,
438
- request_id: str,
439
- prompt: Optional[PromptType] = None,
440
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
441
- arrival_time: Optional[float] = None,
442
- lora_request: Optional[LoRARequest] = None,
443
- trace_headers: Optional[Mapping[str, str]] = None,
444
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
445
- *,
446
- inputs: Optional[PromptType] = None, # DEPRECATED
414
) -> None:
415
"""Async version of :meth:`add_request`."""
449
- if inputs is not None:
450
- prompt = inputs
451
- assert prompt is not None and params is not None
452
-
416
if lora_request is not None and not self.lora_config:
417
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
418
"not enabled!")
@@ -457,7 +420,7 @@ class _AsyncLLMEngine(LLMEngine):
420
arrival_time = time.time()
421
422
preprocessed_inputs = await self.input_preprocessor.preprocess_async(
460
- prompt,
423
+ inputs,
424
request_id=request_id,
425
lora_request=lora_request,
426
prompt_adapter_request=prompt_adapter_request,
@@ -811,55 +774,16 @@ class AsyncLLMEngine:
774
775
# This method does not need to be async, but kept that way
776
# for backwards compatibility.
814
- @overload # DEPRECATED
815
- def add_request(
816
- self,
817
- request_id: str,
818
- *,
819
- inputs: PromptType,
820
- params: Union[SamplingParams, PoolingParams],
821
- arrival_time: Optional[float] = None,
822
- lora_request: Optional[LoRARequest] = None,
823
- trace_headers: Optional[Mapping[str, str]] = None,
824
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
825
- ) -> Coroutine[None, None, AsyncGenerator[Union[
826
- RequestOutput, EmbeddingRequestOutput], None]]:
827
- ...
828
-
829
- @overload
830
- def add_request(
831
- self,
832
- request_id: str,
833
- prompt: PromptType,
834
- params: Union[SamplingParams, PoolingParams],
835
- arrival_time: Optional[float] = None,
836
- lora_request: Optional[LoRARequest] = None,
837
- trace_headers: Optional[Mapping[str, str]] = None,
838
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
839
- ) -> Coroutine[None, None, AsyncGenerator[Union[
840
- RequestOutput, EmbeddingRequestOutput], None]]:
841
- ...
842
-
843
- @deprecate_kwargs(
844
- "inputs",
845
- additional_message="Please use the 'prompt' parameter instead.",
846
- )
777
async def add_request(
778
self,
779
request_id: str,
850
- prompt: Optional[PromptType] = None,
851
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
780
+ inputs: PromptInputs,
781
+ params: Union[SamplingParams, PoolingParams],
782
arrival_time: Optional[float] = None,
783
lora_request: Optional[LoRARequest] = None,
784
trace_headers: Optional[Mapping[str, str]] = None,
855
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
856
- *,
857
- inputs: Optional[PromptType] = None, # DEPRECATED
785
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
786
) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
859
- if inputs is not None:
860
- prompt = inputs
861
- assert prompt is not None and params is not None
862
-
787
if not self.is_running:
788
if self.start_engine_loop:
789
self.start_background_loop()
@@ -873,7 +797,7 @@ class AsyncLLMEngine:
797
stream = self._request_tracker.add_request(
798
request_id,
799
verbose=self.log_requests,
876
- prompt=prompt,
800
+ inputs=inputs,
801
params=params,
802
arrival_time=arrival_time or time.time(),
803
lora_request=lora_request,
@@ -884,7 +808,7 @@ class AsyncLLMEngine:
808
809
async def generate(
810
self,
887
- prompt: PromptType,
811
+ inputs: PromptInputs,
812
sampling_params: SamplingParams,
813
request_id: str,
814
lora_request: Optional[LoRARequest] = None,
@@ -898,7 +822,8 @@ class AsyncLLMEngine:
822
from the LLMEngine to the caller.
823
824
Args:
901
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
825
+ inputs: The inputs to the LLM. See
826
+ :class:`~vllm.inputs.PromptInputs`
827
for more details about the format of each input.
828
sampling_params: The sampling parameters of the request.
829
request_id: The unique id of the request.
@@ -956,7 +881,7 @@ class AsyncLLMEngine:
881
"""
882
async for output in await self.add_request(
883
request_id,
959
- prompt,
884
+ inputs,
885
sampling_params,
886
lora_request=lora_request,
887
trace_headers=trace_headers,
@@ -966,7 +891,7 @@ class AsyncLLMEngine:
891
892
async def encode(
893
self,
969
- prompt: PromptType,
894
+ inputs: PromptInputs,
895
pooling_params: PoolingParams,
896
request_id: str,
897
lora_request: Optional[LoRARequest] = None,
@@ -979,7 +904,8 @@ class AsyncLLMEngine:
904
from the LLMEngine to the caller.
905
906
Args:
982
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
907
+ inputs: The inputs to the LLM. See
908
+ :class:`~vllm.inputs.PromptInputs`
909
for more details about the format of each input.
910
pooling_params: The pooling parameters of the request.
911
request_id: The unique id of the request.
@@ -1033,7 +959,7 @@ class AsyncLLMEngine:
959
"""
960
async for output in await self.add_request(
961
request_id,
1036
- prompt,
962
+ inputs,
963
pooling_params,
964
lora_request=lora_request,
965
trace_headers=trace_headers,
vllm/engine/llm_engine.py
+7
-45
@@ -6,7 +6,7 @@ from functools import partial
6
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7
Iterable, List, Mapping, NamedTuple, Optional)
8
from typing import Sequence as GenericSequence
9
-from typing import Set, Type, Union, overload
9
+from typing import Set, Type, Union
10
11
import torch
12
from typing_extensions import TypeVar
@@ -29,7 +29,7 @@ from vllm.executor.executor_base import ExecutorBase
29
from vllm.executor.gpu_executor import GPUExecutor
30
from vllm.executor.ray_utils import initialize_ray_cluster
31
from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32
- InputRegistry, LLMInputs, PromptType)
32
+ InputRegistry, LLMInputs, PromptInputs)
33
from vllm.inputs.preprocess import InputPreprocessor
34
from vllm.logger import init_logger
35
from vllm.lora.request import LoRARequest
@@ -51,7 +51,7 @@ from vllm.transformers_utils.tokenizer_group import (
51
BaseTokenizerGroup, init_tokenizer_from_configs)
52
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
53
usage_message)
54
-from vllm.utils import Counter, Device, deprecate_kwargs, weak_bind
54
+from vllm.utils import Counter, Device, weak_bind
55
from vllm.version import __version__ as VLLM_VERSION
56
57
logger = init_logger(__name__)
@@ -689,51 +689,16 @@ class LLMEngine:
689
def stop_remote_worker_execution_loop(self) -> None:
690
self.model_executor.stop_remote_worker_execution_loop()
691
692
- @overload # DEPRECATED
692
def add_request(
693
self,
694
request_id: str,
696
- *,
697
- inputs: PromptType,
695
+ inputs: PromptInputs,
696
params: Union[SamplingParams, PoolingParams],
697
arrival_time: Optional[float] = None,
698
lora_request: Optional[LoRARequest] = None,
699
trace_headers: Optional[Mapping[str, str]] = None,
700
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
701
priority: int = 0,
704
- ) -> None:
705
- ...
706
-
707
- @overload
708
- def add_request(
709
- self,
710
- request_id: str,
711
- prompt: PromptType,
712
- params: Union[SamplingParams, PoolingParams],
713
- arrival_time: Optional[float] = None,
714
- lora_request: Optional[LoRARequest] = None,
715
- trace_headers: Optional[Mapping[str, str]] = None,
716
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
717
- priority: int = 0,
718
- ) -> None:
719
- ...
720
-
721
- @deprecate_kwargs(
722
- "inputs",
723
- additional_message="Please use the 'prompt' parameter instead.",
724
- )
725
- def add_request(
726
- self,
727
- request_id: str,
728
- prompt: Optional[PromptType] = None,
729
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
730
- arrival_time: Optional[float] = None,
731
- lora_request: Optional[LoRARequest] = None,
732
- trace_headers: Optional[Mapping[str, str]] = None,
733
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
734
- priority: int = 0,
735
- *,
736
- inputs: Optional[PromptType] = None, # DEPRECATED
702
) -> None:
703
"""Add a request to the engine's request pool.
704
@@ -743,7 +708,8 @@ class LLMEngine:
708
709
Args:
710
request_id: The unique ID of the request.
746
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
711
+ inputs: The inputs to the LLM. See
712
+ :class:`~vllm.inputs.PromptInputs`
713
for more details about the format of each input.
714
params: Parameters for sampling or pooling.
715
:class:`~vllm.SamplingParams` for text generation.
@@ -778,10 +744,6 @@ class LLMEngine:
744
>>> # continue the request processing
745
>>> ...
746
"""
781
- if inputs is not None:
782
- prompt = inputs
783
- assert prompt is not None and params is not None
784
-
747
if lora_request is not None and not self.lora_config:
748
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
749
"not enabled!")
@@ -794,7 +756,7 @@ class LLMEngine:
756
arrival_time = time.time()
757
758
preprocessed_inputs = self.input_preprocessor.preprocess(
797
- prompt,
759
+ inputs,
760
request_id=request_id,
761
lora_request=lora_request,
762
prompt_adapter_request=prompt_adapter_request,
vllm/engine/multiprocessing/__init__.py
+3
-58
@@ -1,14 +1,13 @@
1
from dataclasses import dataclass
2
from enum import Enum
3
-from typing import List, Mapping, Optional, Union, overload
3
+from typing import List, Mapping, Optional, Union
4
5
from vllm import PoolingParams
6
-from vllm.inputs import PromptType
6
+from vllm.inputs import PromptInputs
7
from vllm.lora.request import LoRARequest
8
from vllm.outputs import RequestOutput
9
from vllm.prompt_adapter.request import PromptAdapterRequest
10
from vllm.sampling_params import SamplingParams
11
-from vllm.utils import deprecate_kwargs
11
12
VLLM_RPC_SUCCESS_STR = "SUCCESS"
13
@@ -24,67 +23,13 @@ class MQEngineDeadError(RuntimeError):
23
24
@dataclass
25
class RPCProcessRequest:
27
- prompt: PromptType
26
+ inputs: PromptInputs
27
params: Union[SamplingParams, PoolingParams]
28
request_id: str
29
lora_request: Optional[LoRARequest] = None
30
trace_headers: Optional[Mapping[str, str]] = None
31
prompt_adapter_request: Optional[PromptAdapterRequest] = None
32
34
- @overload # DEPRECATED
35
- def __init__(
36
- self,
37
- *,
38
- inputs: PromptType,
39
- params: Union[SamplingParams, PoolingParams],
40
- request_id: str,
41
- lora_request: Optional[LoRARequest] = None,
42
- trace_headers: Optional[Mapping[str, str]] = None,
43
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
44
- ) -> None:
45
- ...
46
-
47
- @overload
48
- def __init__(
49
- self,
50
- prompt: PromptType,
51
- params: Union[SamplingParams, PoolingParams],
52
- request_id: str,
53
- lora_request: Optional[LoRARequest] = None,
54
- trace_headers: Optional[Mapping[str, str]] = None,
55
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
56
- ) -> None:
57
- ...
58
-
59
- @deprecate_kwargs(
60
- "inputs",
61
- additional_message="Please use the 'prompt' parameter instead.",
62
- )
63
- def __init__(
64
- self,
65
- prompt: Optional[PromptType] = None,
66
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
67
- request_id: Optional[str] = None,
68
- lora_request: Optional[LoRARequest] = None,
69
- trace_headers: Optional[Mapping[str, str]] = None,
70
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
71
- *,
72
- inputs: Optional[PromptType] = None, # DEPRECATED
73
- ) -> None:
74
- if inputs is not None:
75
- prompt = inputs
76
- assert (prompt is not None and params is not None
77
- and request_id is not None)
78
-
79
- super().__init__()
80
-
81
- self.prompt = prompt
82
- self.params = params
83
- self.request_id = request_id
84
- self.lora_request = lora_request
85
- self.trace_headers = trace_headers
86
- self.prompt_adapter_request = prompt_adapter_request
87
-
33
34
@dataclass
35
class RPCError:
vllm/engine/multiprocessing/client.py
+13
-82
@@ -3,7 +3,7 @@ import copy
3
import pickle
4
from contextlib import contextmanager, suppress
5
from typing import (Any, AsyncGenerator, Dict, Iterator, Mapping, Optional,
6
- Union, overload)
6
+ Union)
7
8
import cloudpickle
9
import zmq
@@ -25,14 +25,13 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25
RPCUProfileRequest)
26
# yapf: enable
27
from vllm.envs import VLLM_RPC_TIMEOUT
28
-from vllm.inputs import PromptType
28
+from vllm.inputs import PromptInputs
29
from vllm.logger import init_logger
30
from vllm.lora.request import LoRARequest
31
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
32
from vllm.prompt_adapter.request import PromptAdapterRequest
33
from vllm.sampling_params import SamplingParams
34
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
35
-from vllm.utils import deprecate_kwargs
35
36
logger = init_logger(__name__)
37
@@ -368,45 +367,14 @@ class MQLLMEngineClient:
367
def dead_error(self) -> BaseException:
368
return ENGINE_DEAD_ERROR(self._errored_with)
369
371
- @overload # DEPRECATED
370
def generate(
371
self,
374
- *,
375
- inputs: PromptType,
372
+ inputs: PromptInputs,
373
sampling_params: SamplingParams,
374
request_id: str,
375
lora_request: Optional[LoRARequest] = None,
376
trace_headers: Optional[Mapping[str, str]] = None,
380
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
381
- ) -> AsyncGenerator[RequestOutput, None]:
382
- ...
383
-
384
- @overload
385
- def generate(
386
- self,
387
- prompt: PromptType,
388
- sampling_params: SamplingParams,
389
- request_id: str,
390
- lora_request: Optional[LoRARequest] = None,
391
- trace_headers: Optional[Mapping[str, str]] = None,
392
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
393
- ) -> AsyncGenerator[RequestOutput, None]:
394
- ...
395
-
396
- @deprecate_kwargs(
397
- "inputs",
398
- additional_message="Please use the 'prompt' parameter instead.",
399
- )
400
- def generate(
401
- self,
402
- prompt: Optional[PromptType] = None,
403
- sampling_params: Optional[SamplingParams] = None,
404
- request_id: Optional[str] = None,
405
- lora_request: Optional[LoRARequest] = None,
406
- trace_headers: Optional[Mapping[str, str]] = None,
407
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
408
- *,
409
- inputs: Optional[PromptType] = None # DEPRECATED
377
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
378
) -> AsyncGenerator[RequestOutput, None]:
379
"""Generate outputs for a request.
380
@@ -415,7 +383,8 @@ class MQLLMEngineClient:
383
from the LLMEngine to the caller.
384
385
Args:
418
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
386
+ inputs: The inputs to the LLM. See
387
+ :class:`~vllm.inputs.PromptInputs`
388
for more details about the format of each input.
389
sampling_params: The sampling parameters of the request.
390
request_id: The unique id of the request.
@@ -424,51 +393,17 @@ class MQLLMEngineClient:
393
prompt_adapter_request: Prompt Adapter request to use
394
for generation, if any.
395
"""
427
- if inputs is not None:
428
- prompt = inputs
429
- assert (prompt is not None and sampling_params is not None
430
- and request_id is not None)
431
-
432
- return self._process_request(prompt, sampling_params, request_id,
396
+ return self._process_request(inputs, sampling_params, request_id,
397
lora_request, trace_headers,
398
prompt_adapter_request)
399
436
- @overload # DEPRECATED
400
def encode(
401
self,
439
- *,
440
- inputs: PromptType,
402
+ inputs: PromptInputs,
403
pooling_params: PoolingParams,
404
request_id: str,
405
lora_request: Optional[LoRARequest] = None,
406
trace_headers: Optional[Mapping[str, str]] = None,
445
- ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
446
- ...
447
-
448
- @overload
449
- def encode(
450
- self,
451
- prompt: PromptType,
452
- pooling_params: PoolingParams,
453
- request_id: str,
454
- lora_request: Optional[LoRARequest] = None,
455
- trace_headers: Optional[Mapping[str, str]] = None,
456
- ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
457
- ...
458
-
459
- @deprecate_kwargs(
460
- "inputs",
461
- additional_message="Please use the 'prompt' parameter instead.",
462
- )
463
- def encode(
464
- self,
465
- prompt: Optional[PromptType] = None,
466
- pooling_params: Optional[PoolingParams] = None,
467
- request_id: Optional[str] = None,
468
- lora_request: Optional[LoRARequest] = None,
469
- trace_headers: Optional[Mapping[str, str]] = None,
470
- *,
471
- inputs: Optional[PromptType] = None # DEPRECATED
407
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
408
"""Generate outputs for a request from an embedding model.
409
@@ -477,7 +412,8 @@ class MQLLMEngineClient:
412
from the LLMEngine to the caller.
413
414
Args:
480
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
415
+ inputs: The inputs to the LLM. See
416
+ :class:`~vllm.inputs.PromptInputs`
417
for more details about the format of each input.
418
pooling_params: The pooling parameters of the request.
419
request_id: The unique id of the request.
@@ -488,17 +424,12 @@ class MQLLMEngineClient:
424
The output `EmbeddingRequestOutput` objects from the LLMEngine
425
for the request.
426
"""
491
- if inputs is not None:
492
- prompt = inputs
493
- assert (prompt is not None and pooling_params is not None
494
- and request_id is not None)
495
-
496
- return self._process_request(prompt, pooling_params, request_id,
427
+ return self._process_request(inputs, pooling_params, request_id,
428
lora_request, trace_headers)
429
430
async def _process_request(
431
self,
501
- prompt: PromptType,
432
+ inputs: PromptInputs,
433
params: Union[SamplingParams, PoolingParams],
434
request_id: str,
435
lora_request: Optional[LoRARequest] = None,
@@ -531,7 +462,7 @@ class MQLLMEngineClient:
462
463
request_bytes = pickle.dumps(
464
RPCProcessRequest(
534
- prompt=prompt,
465
+ inputs=inputs,
466
params=params,
467
request_id=request_id,
468
lora_request=lora_request,
vllm/engine/multiprocessing/engine.py
+1
-1
@@ -278,7 +278,7 @@ class MQLLMEngine:
278
try:
279
self.engine.add_request(
280
request_id=request_id,
281
- prompt=request.prompt,
281
+ inputs=request.inputs,
282
params=request.params,
283
lora_request=request.lora_request,
284
trace_headers=request.trace_headers,
vllm/engine/protocol.py
+4
-4
@@ -3,7 +3,7 @@ from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
3
4
from vllm.config import DecodingConfig, ModelConfig
5
from vllm.core.scheduler import SchedulerOutputs
6
-from vllm.inputs.data import PromptType
6
+from vllm.inputs.data import PromptInputs
7
from vllm.lora.request import LoRARequest
8
from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -35,19 +35,19 @@ class EngineClient(Protocol):
35
36
def generate(
37
self,
38
- prompt: PromptType,
38
+ inputs: PromptInputs,
39
sampling_params: SamplingParams,
40
request_id: str,
41
lora_request: Optional[LoRARequest] = None,
42
trace_headers: Optional[Mapping[str, str]] = None,
43
prompt_adapter_request: Optional[PromptAdapterRequest] = None
44
) -> AsyncGenerator[RequestOutput, None]:
45
- """Generate outputs for a request."""
45
+ """Generates outputs for a request"""
46
...
47
48
def encode(
49
self,
50
- prompt: PromptType,
50
+ inputs: PromptInputs,
51
pooling_params: PoolingParams,
52
request_id: str,
53
lora_request: Optional[LoRARequest] = None,
vllm/entrypoints/llm.py
+33
-35
@@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
12
apply_hf_chat_template,
13
apply_mistral_chat_template,
14
parse_chat_messages)
15
-from vllm.inputs import PromptType, TextPrompt, TokensPrompt
15
+from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
16
from vllm.inputs.parse import parse_and_batch_prompt
17
from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
@@ -293,8 +293,8 @@ class LLM:
293
@overload
294
def generate(
295
self,
296
- prompts: Union[PromptType, Sequence[PromptType]],
297
- /,
296
+ inputs: Union[PromptInputs, Sequence[PromptInputs]],
297
+ /, # We may enable `inputs` keyword after removing the old API
298
*,
299
sampling_params: Optional[Union[SamplingParams,
300
Sequence[SamplingParams]]] = None,
@@ -304,13 +304,14 @@ class LLM:
304
...
305
306
@deprecate_kwargs(
307
+ "prompts",
308
"prompt_token_ids",
309
is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
309
- additional_message="Please use the 'prompts' parameter instead.",
310
+ additional_message="Please use the 'inputs' parameter instead.",
311
)
312
def generate(
313
self,
313
- prompts: Union[Union[PromptType, Sequence[PromptType]],
314
+ prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
315
Optional[Union[str, List[str]]]] = None,
316
sampling_params: Optional[Union[SamplingParams,
317
Sequence[SamplingParams]]] = None,
@@ -329,9 +330,7 @@ class LLM:
330
into a single list and pass it to this method.
331
332
Args:
332
- prompts: The prompts to the LLM. You may pass a sequence of prompts
333
- for batch inference. See :class:`~vllm.inputs.PromptType`
334
- for more details about the format of each prompts.
333
+ inputs: A list of inputs to generate completions for.
334
sampling_params: The sampling parameters for text generation. If
335
None, we use the default sampling parameters.
336
When it is a single value, it is applied to every prompt.
@@ -359,13 +358,12 @@ class LLM:
358
"models (XForCausalLM, XForConditionalGeneration).")
359
360
if prompt_token_ids is not None:
362
- parsed_prompts = self._convert_v1_inputs(
361
+ inputs = self._convert_v1_inputs(
362
prompts=cast(Optional[Union[str, List[str]]], prompts),
363
prompt_token_ids=prompt_token_ids,
364
)
365
else:
367
- parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
368
- prompts)
366
+ inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
367
368
if isinstance(guided_options_request, dict):
369
if len(guided_options_request) > 1:
@@ -380,7 +378,7 @@ class LLM:
378
sampling_params = SamplingParams()
379
380
self._validate_and_add_requests(
383
- prompts=parsed_prompts,
381
+ inputs=inputs,
382
params=sampling_params,
383
lora_request=lora_request,
384
prompt_adapter_request=prompt_adapter_request,
@@ -650,8 +648,8 @@ class LLM:
648
@overload
649
def encode(
650
self,
653
- prompts: Union[PromptType, Sequence[PromptType]],
654
- /,
651
+ inputs: Union[PromptInputs, Sequence[PromptInputs]],
652
+ /, # We may enable `inputs` keyword after removing the old API
653
*,
654
pooling_params: Optional[Union[PoolingParams,
655
Sequence[PoolingParams]]] = None,
@@ -661,13 +659,14 @@ class LLM:
659
...
660
661
@deprecate_kwargs(
662
+ "prompts",
663
"prompt_token_ids",
664
is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
666
- additional_message="Please use the 'prompts' parameter instead.",
665
+ additional_message="Please use the 'inputs' parameter instead.",
666
)
667
def encode(
668
self,
670
- prompts: Union[Union[PromptType, Sequence[PromptType]],
669
+ prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
670
Optional[Union[str, List[str]]]] = None,
671
pooling_params: Optional[Union[PoolingParams,
672
Sequence[PoolingParams]]] = None,
@@ -683,9 +682,9 @@ class LLM:
682
into a single list and pass it to this method.
683
684
Args:
686
- prompts: The prompts to the LLM. You may pass a sequence of prompts
687
- for batch inference. See :class:`~vllm.inputs.PromptType`
688
- for more details about the format of each prompts.
685
+ inputs: The inputs to the LLM. You may pass a sequence of inputs for
686
+ batch inference. See :class:`~vllm.inputs.PromptInputs`
687
+ for more details about the format of each input.
688
pooling_params: The pooling parameters for pooling. If None, we
689
use the default pooling parameters.
690
use_tqdm: Whether to use tqdm to display the progress bar.
@@ -708,20 +707,19 @@ class LLM:
707
)
708
709
if prompt_token_ids is not None:
711
- parsed_prompts = self._convert_v1_inputs(
710
+ inputs = self._convert_v1_inputs(
711
prompts=cast(Optional[Union[str, List[str]]], prompts),
712
prompt_token_ids=prompt_token_ids,
713
)
714
else:
716
- parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
717
- prompts)
715
+ inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
716
717
if pooling_params is None:
718
# Use default pooling params.
719
pooling_params = PoolingParams()
720
721
self._validate_and_add_requests(
724
- prompts=parsed_prompts,
722
+ inputs=inputs,
723
params=pooling_params,
724
lora_request=lora_request,
725
prompt_adapter_request=prompt_adapter_request,
@@ -765,9 +763,9 @@ class LLM:
763
raise ValueError("Either prompts or prompt_token_ids must be "
764
"provided.")
765
768
- parsed_prompts: List[PromptType] = []
766
+ inputs: List[PromptInputs] = []
767
for i in range(num_requests):
770
- item: PromptType
768
+ item: PromptInputs
769
770
if prompts is not None:
771
item = TextPrompt(prompt=prompts[i])
@@ -776,13 +774,13 @@ class LLM:
774
else:
775
raise AssertionError
776
779
- parsed_prompts.append(item)
777
+ inputs.append(item)
778
781
- return parsed_prompts
779
+ return inputs
780
781
def _validate_and_add_requests(
782
self,
785
- prompts: Union[PromptType, Sequence[PromptType]],
783
+ inputs: Union[PromptInputs, Sequence[PromptInputs]],
784
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
785
Sequence[PoolingParams]],
786
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
@@ -790,11 +788,11 @@ class LLM:
788
guided_options: Optional[GuidedDecodingRequest] = None,
789
priority: Optional[List[int]] = None,
790
) -> None:
793
- if isinstance(prompts, (str, dict)):
791
+ if isinstance(inputs, (str, dict)):
792
# Convert a single prompt to a list.
795
- prompts = [prompts]
793
+ inputs = [inputs]
794
797
- num_requests = len(prompts)
795
+ num_requests = len(inputs)
796
if isinstance(params, list) and len(params) != num_requests:
797
raise ValueError("The lengths of prompts and params "
798
"must be the same.")
@@ -811,9 +809,9 @@ class LLM:
809
sp.output_kind = RequestOutputKind.FINAL_ONLY
810
811
# Add requests to the engine.
814
- for i, prompt in enumerate(prompts):
812
+ for i, request_inputs in enumerate(inputs):
813
self._add_request(
816
- prompt,
814
+ request_inputs,
815
params[i] if isinstance(params, Sequence) else params,
816
lora_request=lora_request[i] if isinstance(
817
lora_request, Sequence) else lora_request,
@@ -823,7 +821,7 @@ class LLM:
821
822
def _add_request(
823
self,
826
- prompt: PromptType,
824
+ inputs: PromptInputs,
825
params: Union[SamplingParams, PoolingParams],
826
lora_request: Optional[LoRARequest] = None,
827
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -832,7 +830,7 @@ class LLM:
830
request_id = str(next(self.request_counter))
831
self.llm_engine.add_request(
832
request_id,
835
- prompt,
833
+ inputs,
834
params,
835
lora_request=lora_request,
836
prompt_adapter_request=prompt_adapter_request,
vllm/inputs/__init__.py
+3
-17
@@ -1,5 +1,5 @@
1
from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2
- LLMInputs, PromptType, SingletonPrompt, TextPrompt,
2
+ LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
3
TokensPrompt, build_explicit_enc_dec_prompt,
4
to_enc_dec_tuple_list, zip_enc_dec_prompts)
5
from .registry import InputContext, InputRegistry
@@ -16,8 +16,8 @@ See also:
16
__all__ = [
17
"TextPrompt",
18
"TokensPrompt",
19
- "PromptType",
20
- "SingletonPrompt",
19
+ "PromptInputs",
20
+ "SingletonPromptInputs",
21
"ExplicitEncoderDecoderPrompt",
22
"LLMInputs",
23
"EncoderDecoderLLMInputs",
@@ -28,17 +28,3 @@ __all__ = [
28
"InputContext",
29
"InputRegistry",
30
]
31
-
32
-
33
-def __getattr__(name: str):
34
- if name == "PromptInput":
35
- import warnings
36
-
37
- msg = ("PromptInput has been renamed to PromptType. "
38
- "The original name will be removed in an upcoming version.")
39
-
40
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
41
-
42
- return PromptType
43
-
44
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/data.py
+19
-29
@@ -33,7 +33,7 @@ class TokensPrompt(TypedDict):
33
"""
34
35
36
-SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
36
+SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
37
"""
38
Set of possible schemas for a single LLM input:
39
@@ -46,7 +46,7 @@ which may be utilized for encoder/decoder models when
46
the user desires to express both the encoder & decoder
47
prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
49
-A prompt of type :class:`SingletonPrompt` may be employed
49
+A prompt of type :class:`SingletonPromptInputs` may be employed
50
as (1) input to a decoder-only model, (2) input to
51
the encoder of an encoder/decoder model, in the scenario
52
where the decoder-prompt is not specified explicitly, or
@@ -55,33 +55,33 @@ more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55
"""
56
57
_T1_co = TypeVar("_T1_co",
58
- bound=SingletonPrompt,
59
- default=SingletonPrompt,
58
+ bound=SingletonPromptInputs,
59
+ default=SingletonPromptInputs,
60
covariant=True)
61
_T2_co = TypeVar("_T2_co",
62
- bound=SingletonPrompt,
63
- default=SingletonPrompt,
62
+ bound=SingletonPromptInputs,
63
+ default=SingletonPromptInputs,
64
covariant=True)
65
66
67
# TODO: Make fields ReadOnly once mypy supports it
68
class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
69
- """
70
- Represents an encoder/decoder model input prompt,
71
- comprising an explicit encoder prompt and a decoder prompt.
69
+ """Represents an encoder/decoder model input prompt,
70
+ comprising an explicit encoder prompt and a
71
+ decoder prompt.
72
73
The encoder and decoder prompts, respectively,
74
may formatted according to any of the
75
- :class:`SingletonPrompt` schemas, and are not
75
+ :class:`SingletonPromptInputs` schemas, and are not
76
required to have the same schema.
77
78
Only the encoder prompt may have multi-modal data.
79
80
Note that an :class:`ExplicitEncoderDecoderPrompt` may not
81
be used as an input to a decoder-only model,
82
- and that the :code:`encoder_prompt` and :code:`decoder_prompt`
82
+ and that the `encoder_prompt` and `decoder_prompt`
83
fields of this data structure themselves must be
84
- :class:`SingletonPrompt` instances.
84
+ :class:`SingletonPromptInputs` instances.
85
"""
86
87
encoder_prompt: _T1_co
@@ -89,7 +89,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
89
decoder_prompt: Optional[_T2_co]
90
91
92
-PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
92
+PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
93
"""
94
Set of possible schemas for an LLM input, including
95
both decoder-only and encoder/decoder input types:
@@ -140,8 +140,12 @@ class EncoderDecoderLLMInputs(LLMInputs):
140
"""
141
142
143
-_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
144
-_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
143
+_T1 = TypeVar("_T1",
144
+ bound=SingletonPromptInputs,
145
+ default=SingletonPromptInputs)
146
+_T2 = TypeVar("_T2",
147
+ bound=SingletonPromptInputs,
148
+ default=SingletonPromptInputs)
149
150
151
def build_explicit_enc_dec_prompt(
@@ -172,17 +176,3 @@ def to_enc_dec_tuple_list(
176
return [(enc_dec_prompt["encoder_prompt"],
177
enc_dec_prompt["decoder_prompt"])
178
for enc_dec_prompt in enc_dec_prompts]
175
-
176
-
177
-def __getattr__(name: str):
178
- if name == "PromptInput":
179
- import warnings
180
-
181
- msg = ("PromptInput has been renamed to PromptType. "
182
- "The original name will be removed in an upcoming version.")
183
-
184
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
185
-
186
- return PromptType
187
-
188
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/parse.py
+11
-11
@@ -5,7 +5,7 @@ from typing_extensions import TypeIs
5
from vllm.utils import is_list_of
6
7
from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8
- LLMInputs, PromptType, SingletonPrompt, TextPrompt,
8
+ LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
9
TokensPrompt)
10
11
@@ -81,23 +81,23 @@ class ParsedTokensPrompt(TypedDict):
81
82
83
def parse_singleton_prompt(
84
- prompt: SingletonPrompt,
84
+ inputs: SingletonPromptInputs,
85
) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86
- if isinstance(prompt, str):
87
- return ParsedStrPrompt(type="str", content=prompt)
88
- elif isinstance(prompt, dict):
89
- if "prompt_token_ids" in prompt:
86
+ if isinstance(inputs, str):
87
+ return ParsedStrPrompt(type="str", content=inputs)
88
+ elif isinstance(inputs, dict):
89
+ if "prompt_token_ids" in inputs:
90
return ParsedTokensPrompt(type="tokens",
91
- content=prompt) # type: ignore
92
- elif "prompt" in prompt:
93
- return ParsedTextPrompt(type="text", content=prompt)
91
+ content=inputs) # type: ignore
92
+ elif "prompt" in inputs:
93
+ return ParsedTextPrompt(type="text", content=inputs)
94
95
raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98
def is_explicit_encoder_decoder_prompt(
99
- prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100
- return isinstance(prompt, dict) and "encoder_prompt" in prompt
99
+ inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100
+ return isinstance(inputs, dict) and "encoder_prompt" in inputs
101
102
103
def is_valid_encoder_decoder_llm_inputs(
vllm/inputs/preprocess.py
+43
-43
@@ -9,8 +9,8 @@ from vllm.lora.request import LoRARequest
9
from vllm.prompt_adapter.request import PromptAdapterRequest
10
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
12
-from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
13
- SingletonPrompt)
12
+from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
13
+ SingletonPromptInputs)
14
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
15
16
if TYPE_CHECKING:
@@ -206,7 +206,7 @@ class InputPreprocessor:
206
207
def _extract_prompt_components(
208
self,
209
- prompt: SingletonPrompt,
209
+ inputs: SingletonPromptInputs,
210
request_id: str,
211
lora_request: Optional[LoRARequest] = None,
212
) -> PromptComponents:
@@ -216,7 +216,7 @@ class InputPreprocessor:
216
Arguments:
217
218
* request_id
219
- * prompt: single encoder or decoder input prompt
219
+ * inputs: single encoder or decoder input prompt
220
* lora_request: this is only valid for decoder prompts
221
222
Returns:
@@ -226,24 +226,24 @@ class InputPreprocessor:
226
* multi_modal_data
227
'''
228
229
- parsed = parse_singleton_prompt(prompt)
229
+ parsed = parse_singleton_prompt(inputs)
230
231
if parsed["type"] == "str":
232
- prompt_text = parsed["content"]
232
+ prompt = parsed["content"]
233
prompt_token_ids = self._tokenize_prompt(
234
- prompt_text,
234
+ prompt,
235
request_id=request_id,
236
lora_request=lora_request,
237
)
238
multi_modal_data = None
239
elif parsed["type"] == "tokens":
240
- prompt_text = None
240
+ prompt = None
241
prompt_token_ids = parsed["content"]["prompt_token_ids"]
242
multi_modal_data = parsed["content"].get("multi_modal_data")
243
elif parsed["type"] == "text":
244
- prompt_text = parsed["content"]["prompt"]
244
+ prompt = parsed["content"]["prompt"]
245
prompt_token_ids = self._tokenize_prompt(
246
- prompt_text,
246
+ prompt,
247
request_id=request_id,
248
lora_request=lora_request,
249
)
@@ -251,33 +251,33 @@ class InputPreprocessor:
251
else:
252
assert_never(parsed)
253
254
- return prompt_text, prompt_token_ids, multi_modal_data
254
+ return prompt, prompt_token_ids, multi_modal_data
255
256
async def _extract_prompt_components_async(
257
self,
258
- prompt: SingletonPrompt,
258
+ inputs: SingletonPromptInputs,
259
request_id: str,
260
lora_request: Optional[LoRARequest] = None,
261
) -> PromptComponents:
262
"""Async version of :meth:`_extract_prompt_components`."""
263
- parsed = parse_singleton_prompt(prompt)
263
+ parsed = parse_singleton_prompt(inputs)
264
265
if parsed["type"] == "str":
266
- prompt_text = parsed["content"]
266
+ prompt = parsed["content"]
267
prompt_token_ids = await self._tokenize_prompt_async(
268
- prompt_text,
268
+ prompt,
269
request_id=request_id,
270
lora_request=lora_request,
271
)
272
multi_modal_data = None
273
elif parsed["type"] == "tokens":
274
- prompt_text = None
274
+ prompt = None
275
prompt_token_ids = parsed["content"]["prompt_token_ids"]
276
multi_modal_data = parsed["content"].get("multi_modal_data")
277
elif parsed["type"] == "text":
278
- prompt_text = parsed["content"]["prompt"]
278
+ prompt = parsed["content"]["prompt"]
279
prompt_token_ids = await self._tokenize_prompt_async(
280
- prompt_text,
280
+ prompt,
281
request_id=request_id,
282
lora_request=lora_request,
283
)
@@ -285,7 +285,7 @@ class InputPreprocessor:
285
else:
286
assert_never(parsed)
287
288
- return prompt_text, prompt_token_ids, multi_modal_data
288
+ return prompt, prompt_token_ids, multi_modal_data
289
290
def _build_enc_dec_llm_inputs(
291
self,
@@ -311,7 +311,7 @@ class InputPreprocessor:
311
312
def _process_encoder_decoder_prompt(
313
self,
314
- prompt: PromptType,
314
+ inputs: PromptInputs,
315
request_id: str,
316
) -> EncoderDecoderLLMInputs:
317
'''
@@ -339,7 +339,7 @@ class InputPreprocessor:
339
340
Arguments:
341
342
- * prompt: an input prompt
342
+ * inputs: an input prompt
343
* request_id
344
345
Returns:
@@ -350,13 +350,13 @@ class InputPreprocessor:
350
encoder_comps: PromptComponents
351
decoder_comps: DecoderPromptComponents
352
353
- if is_explicit_encoder_decoder_prompt(prompt):
353
+ if is_explicit_encoder_decoder_prompt(inputs):
354
encoder_comps = self._extract_prompt_components(
355
- prompt["encoder_prompt"],
355
+ inputs["encoder_prompt"],
356
request_id=request_id,
357
)
358
359
- if (decoder_input := prompt["decoder_prompt"]) is None:
359
+ if (decoder_input := inputs["decoder_prompt"]) is None:
360
decoder_comps = None, None, None
361
else:
362
decoder_comps = self._extract_prompt_components(
@@ -365,7 +365,7 @@ class InputPreprocessor:
365
)
366
else:
367
encoder_comps = self._extract_prompt_components(
368
- prompt,
368
+ inputs,
369
request_id=request_id,
370
)
371
@@ -375,20 +375,20 @@ class InputPreprocessor:
375
376
async def _process_encoder_decoder_prompt_async(
377
self,
378
- prompt: PromptType,
378
+ inputs: PromptInputs,
379
request_id: str,
380
) -> EncoderDecoderLLMInputs:
381
"""Async version of :meth:`_process_encoder_decoder_prompt`."""
382
encoder_comps: PromptComponents
383
decoder_comps: DecoderPromptComponents
384
385
- if is_explicit_encoder_decoder_prompt(prompt):
385
+ if is_explicit_encoder_decoder_prompt(inputs):
386
encoder_task = self._extract_prompt_components_async(
387
- prompt["encoder_prompt"],
387
+ inputs["encoder_prompt"],
388
request_id=request_id,
389
)
390
391
- if (decoder_input := prompt["decoder_prompt"]) is None:
391
+ if (decoder_input := inputs["decoder_prompt"]) is None:
392
encoder_comps = await encoder_task
393
decoder_comps = None, None, None
394
else:
@@ -401,7 +401,7 @@ class InputPreprocessor:
401
encoder_task, decoder_task)
402
else:
403
encoder_comps = await self._extract_prompt_components_async(
404
- prompt,
404
+ inputs,
405
request_id=request_id,
406
)
407
@@ -425,7 +425,7 @@ class InputPreprocessor:
425
426
def _process_decoder_only_prompt(
427
self,
428
- prompt: SingletonPrompt,
428
+ inputs: SingletonPromptInputs,
429
request_id: str,
430
lora_request: Optional[LoRARequest] = None,
431
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -436,7 +436,7 @@ class InputPreprocessor:
436
437
Arguments:
438
439
- * prompt: input prompt
439
+ * inputs: input prompt
440
* request_id
441
* lora_request
442
* prompt_adapter_request
@@ -447,7 +447,7 @@ class InputPreprocessor:
447
'''
448
449
prompt_comps = self._extract_prompt_components(
450
- prompt,
450
+ inputs,
451
request_id=request_id,
452
lora_request=lora_request,
453
)
@@ -459,14 +459,14 @@ class InputPreprocessor:
459
460
async def _process_decoder_only_prompt_async(
461
self,
462
- prompt: SingletonPrompt,
462
+ inputs: SingletonPromptInputs,
463
request_id: str,
464
lora_request: Optional[LoRARequest] = None,
465
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
466
) -> LLMInputs:
467
"""Async version of :meth:`_process_decoder_only_prompt`."""
468
prompt_comps = await self._extract_prompt_components_async(
469
- prompt,
469
+ inputs,
470
request_id=request_id,
471
lora_request=lora_request,
472
)
@@ -478,7 +478,7 @@ class InputPreprocessor:
478
479
def preprocess(
480
self,
481
- prompt: PromptType,
481
+ inputs: PromptInputs,
482
request_id: str,
483
lora_request: Optional[LoRARequest] = None,
484
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -488,17 +488,17 @@ class InputPreprocessor:
488
# Encoder-decoder model requires special mapping of
489
# input prompts to encoder & decoder
490
return self._process_encoder_decoder_prompt(
491
- prompt,
491
+ inputs,
492
request_id=request_id,
493
)
494
495
- if is_explicit_encoder_decoder_prompt(prompt):
495
+ if is_explicit_encoder_decoder_prompt(inputs):
496
raise ValueError("Cannot pass encoder-decoder prompt "
497
"to decoder-only models")
498
499
# Decoder-only operation
500
return self._process_decoder_only_prompt(
501
- prompt,
501
+ inputs,
502
request_id=request_id,
503
lora_request=lora_request,
504
prompt_adapter_request=prompt_adapter_request,
@@ -506,7 +506,7 @@ class InputPreprocessor:
506
507
async def preprocess_async(
508
self,
509
- prompt: PromptType,
509
+ inputs: PromptInputs,
510
request_id: str,
511
lora_request: Optional[LoRARequest] = None,
512
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -516,17 +516,17 @@ class InputPreprocessor:
516
# Encoder-decoder model requires special mapping of
517
# input prompts to encoder & decoder
518
return await self._process_encoder_decoder_prompt_async(
519
- prompt,
519
+ inputs,
520
request_id=request_id,
521
)
522
523
- if is_explicit_encoder_decoder_prompt(prompt):
523
+ if is_explicit_encoder_decoder_prompt(inputs):
524
raise ValueError("Cannot pass encoder-decoder prompt "
525
"to decoder-only models")
526
527
# Decoder-only operation
528
return await self._process_decoder_only_prompt_async(
529
- prompt,
529
+ inputs,
530
request_id=request_id,
531
lora_request=lora_request,
532
prompt_adapter_request=prompt_adapter_request,