@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b00b9c26c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] rename`PromptInputs` and `inputs` (#8876)

Cyrus Leung committed Sep 27, 2024 at 11:35 UTC 3b00b9c26c91e9f9ada12975b613555698054e39
21 files changed +440 -248
benchmarks/benchmark_latency.py
+4 -4
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
13 from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14 -from vllm.inputs import PromptInputs
14 +from vllm.inputs import PromptType
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61 dummy_prompt_token_ids = np.random.randint(10000,
62 size=(args.batch_size,
63 args.input_len))
64 - dummy_inputs: List[PromptInputs] = [{
64 + dummy_prompts: List[PromptType] = [{
65 "prompt_token_ids": batch
66 } for batch in dummy_prompt_token_ids.tolist()]
67
@@ -74,13 +74,13 @@ def main(args: argparse.Namespace):
74 ],
75 on_trace_ready=torch.profiler.tensorboard_trace_handler(
76 str(profile_dir))) as p:
77 - llm.generate(dummy_inputs,
77 + llm.generate(dummy_prompts,
78 sampling_params=sampling_params,
79 use_tqdm=False)
80 print(p.key_averages())
81 else:
82 start_time = time.perf_counter()
83 - llm.generate(dummy_inputs,
83 + llm.generate(dummy_prompts,
84 sampling_params=sampling_params,
85 use_tqdm=False)
86 end_time = time.perf_counter()
docs/source/dev/multimodal/multimodal_index.rst
+1 -1
@@ -8,7 +8,7 @@ Multi-Modality
8 vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10 Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11 -via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
11 +via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
12
13 Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14 by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1 -1
@@ -1,7 +1,7 @@
1 LLM Inputs
2 ==========
3
4 -.. autodata:: vllm.inputs.PromptInputs
4 +.. autodata:: vllm.inputs.PromptType
5
6 .. autoclass:: vllm.inputs.TextPrompt
7 :show-inheritance:
docs/source/models/vlm.rst
+1 -1
@@ -27,7 +27,7 @@ The :class:`~vllm.LLM` class can be instantiated in much the same way as languag
27 We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28 the above snippet. Specifically, ``image_feature_size`` can no longer be specified as we now calculate that internally for each model.
29
30 -To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
30 +To pass an image to the model, note the following in :class:`vllm.inputs.PromptType`:
31
32 * ``prompt``: The prompt should follow the format that is documented on HuggingFace.
33 * ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/async_engine/test_async_llm_engine.py
+5 -3
@@ -86,17 +86,19 @@ class MockAsyncLLMEngine(AsyncLLMEngine):
86
87 @pytest.mark.asyncio
88 async def test_new_requests_event():
89 + params = SamplingParams()
90 +
91 engine = MockAsyncLLMEngine()
92 engine.start_background_loop()
93 await asyncio.sleep(0.01)
94 assert engine.engine.step_calls == 0
95
94 - await engine.add_request("1", "", None)
96 + await engine.add_request("1", "", params)
97 await asyncio.sleep(0.01)
98 assert engine.engine.add_request_calls == 1
99 assert engine.engine.step_calls == 1
100
99 - await engine.add_request("2", "", None)
101 + await engine.add_request("2", "", params)
102 engine.engine.generate("2")
103 await asyncio.sleep(0)
104 await asyncio.sleep(0)
@@ -111,7 +113,7 @@ async def test_new_requests_event():
113 await asyncio.sleep(0.001)
114 assert engine.engine.step_calls == old_step_calls
115
114 - await engine.add_request("3", "", None)
116 + await engine.add_request("3", "", params)
117 await asyncio.sleep(0.01)
118 assert engine.engine.add_request_calls == 3
119 assert engine.engine.step_calls == old_step_calls + 1
tests/entrypoints/llm/test_encode.py
-34
@@ -49,21 +49,6 @@ def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
49 assert [o.outputs for o in o1] == [o.outputs for o in o2]
50
51
52 -@pytest.mark.skip_global_cleanup
53 -@pytest.mark.parametrize('prompt', PROMPTS)
54 -def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
55 - pooling_params = PoolingParams()
56 -
57 - with pytest.warns(DeprecationWarning, match="'prompts'"):
58 - v1_output = llm.encode(prompts=prompt, pooling_params=pooling_params)
59 -
60 - v2_output = llm.encode(prompt, pooling_params=pooling_params)
61 - assert_outputs_equal(v1_output, v2_output)
62 -
63 - v2_output = llm.encode({"prompt": prompt}, pooling_params=pooling_params)
64 - assert_outputs_equal(v1_output, v2_output)
65 -
66 -
52 @pytest.mark.skip_global_cleanup
53 @pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
54 def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -79,25 +64,6 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
64 assert_outputs_equal(v1_output, v2_output)
65
66
82 -@pytest.mark.skip_global_cleanup
83 -def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84 - pooling_params = PoolingParams()
85 -
86 - with pytest.warns(DeprecationWarning, match="'prompts'"):
87 - v1_output = llm.encode(prompts=PROMPTS, pooling_params=pooling_params)
88 -
89 - v2_output = llm.encode(PROMPTS, pooling_params=pooling_params)
90 - assert_outputs_equal(v1_output, v2_output)
91 -
92 - v2_output = llm.encode(
93 - [{
94 - "prompt": p
95 - } for p in PROMPTS],
96 - pooling_params=pooling_params,
97 - )
98 - assert_outputs_equal(v1_output, v2_output)
99 -
100 -
67 @pytest.mark.skip_global_cleanup
68 def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
69 pooling_params = PoolingParams()
tests/entrypoints/llm/test_generate.py
-37
@@ -47,23 +47,6 @@ def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
47 assert [o.outputs for o in o1] == [o.outputs for o in o2]
48
49
50 -@pytest.mark.skip_global_cleanup
51 -@pytest.mark.parametrize('prompt', PROMPTS)
52 -def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
53 - sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
54 -
55 - with pytest.warns(DeprecationWarning, match="'prompts'"):
56 - v1_output = llm.generate(prompts=prompt,
57 - sampling_params=sampling_params)
58 -
59 - v2_output = llm.generate(prompt, sampling_params=sampling_params)
60 - assert_outputs_equal(v1_output, v2_output)
61 -
62 - v2_output = llm.generate({"prompt": prompt},
63 - sampling_params=sampling_params)
64 - assert_outputs_equal(v1_output, v2_output)
65 -
66 -
50 @pytest.mark.skip_global_cleanup
51 @pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
52 def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -79,26 +62,6 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
62 assert_outputs_equal(v1_output, v2_output)
63
64
82 -@pytest.mark.skip_global_cleanup
83 -def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84 - sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
85 -
86 - with pytest.warns(DeprecationWarning, match="'prompts'"):
87 - v1_output = llm.generate(prompts=PROMPTS,
88 - sampling_params=sampling_params)
89 -
90 - v2_output = llm.generate(PROMPTS, sampling_params=sampling_params)
91 - assert_outputs_equal(v1_output, v2_output)
92 -
93 - v2_output = llm.generate(
94 - [{
95 - "prompt": p
96 - } for p in PROMPTS],
97 - sampling_params=sampling_params,
98 - )
99 - assert_outputs_equal(v1_output, v2_output)
100 -
101 -
65 @pytest.mark.skip_global_cleanup
66 def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
67 sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
tests/mq_llm_engine/test_error_handling.py
+6 -6
@@ -61,7 +61,7 @@ async def test_evil_forward(tmp_socket):
61
62 # Throws an error in first forward pass.
63 with pytest.raises(RAISED_ERROR):
64 - async for _ in client.generate(inputs="Hello my name is",
64 + async for _ in client.generate(prompt="Hello my name is",
65 sampling_params=SamplingParams(),
66 request_id=uuid.uuid4()):
67 pass
@@ -69,7 +69,7 @@ async def test_evil_forward(tmp_socket):
69
70 # Engine is errored, should get ENGINE_DEAD_ERROR.
71 with pytest.raises(MQEngineDeadError):
72 - async for _ in client.generate(inputs="Hello my name is",
72 + async for _ in client.generate(prompt="Hello my name is",
73 sampling_params=SamplingParams(),
74 request_id=uuid.uuid4()):
75 pass
@@ -118,7 +118,7 @@ async def test_failed_health_check(tmp_socket):
118
119 # Generate call should throw ENGINE_DEAD_ERROR
120 with pytest.raises(MQEngineDeadError):
121 - async for _ in client.generate(inputs="Hello my name is",
121 + async for _ in client.generate(prompt="Hello my name is",
122 sampling_params=SamplingParams(),
123 request_id=uuid.uuid4()):
124 pass
@@ -160,7 +160,7 @@ async def test_failed_abort(tmp_socket):
160 # with reference to the original KeyError("foo")
161 with pytest.raises(MQEngineDeadError) as execinfo:
162 async for _ in client.generate(
163 - inputs="Hello my name is",
163 + prompt="Hello my name is",
164 sampling_params=SamplingParams(max_tokens=10),
165 request_id=uuid.uuid4()):
166 pass
@@ -183,7 +183,7 @@ async def test_bad_request(tmp_socket):
183
184 # Invalid request should fail, but not crash the server.
185 with pytest.raises(ValueError):
186 - async for _ in client.generate(inputs="Hello my name is",
186 + async for _ in client.generate(prompt="Hello my name is",
187 sampling_params=SamplingParams(),
188 request_id="abcd-1",
189 lora_request=LoRARequest(
@@ -192,7 +192,7 @@ async def test_bad_request(tmp_socket):
192 pass
193
194 # This request should be okay.
195 - async for _ in client.generate(inputs="Hello my name is",
195 + async for _ in client.generate(prompt="Hello my name is",
196 sampling_params=SamplingParams(),
197 request_id="abcd-2"):
198 pass
tests/mq_llm_engine/utils.py
+1 -1
@@ -20,7 +20,7 @@ async def generate(
20 count = 0
21 async for out in client.generate(
22 request_id=request_id,
23 - inputs="Hello my name is Robert and",
23 + prompt="Hello my name is Robert and",
24 sampling_params=SamplingParams(max_tokens=num_tokens,
25 temperature=0)):
26
vllm/__init__.py
+2 -2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 -from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
8 +from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11 EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19 "__version_tuple__",
20 "LLM",
21 "ModelRegistry",
22 - "PromptInputs",
22 + "PromptType",
23 "TextPrompt",
24 "TokensPrompt",
25 "SamplingParams",
vllm/engine/async_llm_engine.py
+92 -18
@@ -2,8 +2,8 @@ import asyncio
2 import time
3 import weakref
4 from functools import partial
5 -from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6 - Mapping, Optional, Set, Tuple, Type, Union)
5 +from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
6 + List, Mapping, Optional, Set, Tuple, Type, Union, overload)
7 from weakref import ReferenceType
8
9 import vllm.envs as envs
@@ -17,7 +17,7 @@ from vllm.engine.metrics_types import StatLoggerBase
17 from vllm.executor.executor_base import ExecutorAsyncBase
18 from vllm.executor.gpu_executor import GPUExecutorAsync
19 from vllm.executor.ray_utils import initialize_ray_cluster
20 -from vllm.inputs import PromptInputs
20 +from vllm.inputs import PromptType
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -28,7 +28,7 @@ from vllm.sampling_params import SamplingParams
28 from vllm.sequence import ExecuteModelRequest
29 from vllm.transformers_utils.tokenizer import AnyTokenizer
30 from vllm.usage.usage_lib import UsageContext
31 -from vllm.utils import weak_bind
31 +from vllm.utils import deprecate_kwargs, weak_bind
32
33 logger = init_logger(__name__)
34 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -402,17 +402,54 @@ class _AsyncLLMEngine(LLMEngine):
402 """Stop the remote worker execution loop."""
403 await self.model_executor.stop_remote_worker_execution_loop_async()
404
405 + @overload # DEPRECATED
406 async def add_request_async(
407 self,
408 request_id: str,
408 - inputs: PromptInputs,
409 + *,
410 + inputs: PromptType,
411 params: Union[SamplingParams, PoolingParams],
412 arrival_time: Optional[float] = None,
413 lora_request: Optional[LoRARequest] = None,
414 trace_headers: Optional[Mapping[str, str]] = None,
415 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
416 + ) -> None:
417 + ...
418 +
419 + @overload
420 + async def add_request_async(
421 + self,
422 + request_id: str,
423 + prompt: PromptType,
424 + params: Union[SamplingParams, PoolingParams],
425 + arrival_time: Optional[float] = None,
426 + lora_request: Optional[LoRARequest] = None,
427 + trace_headers: Optional[Mapping[str, str]] = None,
428 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
429 + ) -> None:
430 + ...
431 +
432 + @deprecate_kwargs(
433 + "inputs",
434 + additional_message="Please use the 'prompt' parameter instead.",
435 + )
436 + async def add_request_async(
437 + self,
438 + request_id: str,
439 + prompt: Optional[PromptType] = None,
440 + params: Optional[Union[SamplingParams, PoolingParams]] = None,
441 + arrival_time: Optional[float] = None,
442 + lora_request: Optional[LoRARequest] = None,
443 + trace_headers: Optional[Mapping[str, str]] = None,
444 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
445 + *,
446 + inputs: Optional[PromptType] = None, # DEPRECATED
447 ) -> None:
448 """Async version of :meth:`add_request`."""
449 + if inputs is not None:
450 + prompt = inputs
451 + assert prompt is not None and params is not None
452 +
453 if lora_request is not None and not self.lora_config:
454 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
455 "not enabled!")
@@ -420,7 +457,7 @@ class _AsyncLLMEngine(LLMEngine):
457 arrival_time = time.time()
458
459 preprocessed_inputs = await self.input_preprocessor.preprocess_async(
423 - inputs,
460 + prompt,
461 request_id=request_id,
462 lora_request=lora_request,
463 prompt_adapter_request=prompt_adapter_request,
@@ -774,16 +811,55 @@ class AsyncLLMEngine:
811
812 # This method does not need to be async, but kept that way
813 # for backwards compatibility.
777 - async def add_request(
814 + @overload # DEPRECATED
815 + def add_request(
816 self,
817 request_id: str,
780 - inputs: PromptInputs,
818 + *,
819 + inputs: PromptType,
820 params: Union[SamplingParams, PoolingParams],
821 arrival_time: Optional[float] = None,
822 lora_request: Optional[LoRARequest] = None,
823 trace_headers: Optional[Mapping[str, str]] = None,
785 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
824 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
825 + ) -> Coroutine[None, None, AsyncGenerator[Union[
826 + RequestOutput, EmbeddingRequestOutput], None]]:
827 + ...
828 +
829 + @overload
830 + def add_request(
831 + self,
832 + request_id: str,
833 + prompt: PromptType,
834 + params: Union[SamplingParams, PoolingParams],
835 + arrival_time: Optional[float] = None,
836 + lora_request: Optional[LoRARequest] = None,
837 + trace_headers: Optional[Mapping[str, str]] = None,
838 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
839 + ) -> Coroutine[None, None, AsyncGenerator[Union[
840 + RequestOutput, EmbeddingRequestOutput], None]]:
841 + ...
842 +
843 + @deprecate_kwargs(
844 + "inputs",
845 + additional_message="Please use the 'prompt' parameter instead.",
846 + )
847 + async def add_request(
848 + self,
849 + request_id: str,
850 + prompt: Optional[PromptType] = None,
851 + params: Optional[Union[SamplingParams, PoolingParams]] = None,
852 + arrival_time: Optional[float] = None,
853 + lora_request: Optional[LoRARequest] = None,
854 + trace_headers: Optional[Mapping[str, str]] = None,
855 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
856 + *,
857 + inputs: Optional[PromptType] = None, # DEPRECATED
858 ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
859 + if inputs is not None:
860 + prompt = inputs
861 + assert prompt is not None and params is not None
862 +
863 if not self.is_running:
864 if self.start_engine_loop:
865 self.start_background_loop()
@@ -797,7 +873,7 @@ class AsyncLLMEngine:
873 stream = self._request_tracker.add_request(
874 request_id,
875 verbose=self.log_requests,
800 - inputs=inputs,
876 + prompt=prompt,
877 params=params,
878 arrival_time=arrival_time or time.time(),
879 lora_request=lora_request,
@@ -808,7 +884,7 @@ class AsyncLLMEngine:
884
885 async def generate(
886 self,
811 - inputs: PromptInputs,
887 + prompt: PromptType,
888 sampling_params: SamplingParams,
889 request_id: str,
890 lora_request: Optional[LoRARequest] = None,
@@ -822,8 +898,7 @@ class AsyncLLMEngine:
898 from the LLMEngine to the caller.
899
900 Args:
825 - inputs: The inputs to the LLM. See
826 - :class:`~vllm.inputs.PromptInputs`
901 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
902 for more details about the format of each input.
903 sampling_params: The sampling parameters of the request.
904 request_id: The unique id of the request.
@@ -881,7 +956,7 @@ class AsyncLLMEngine:
956 """
957 async for output in await self.add_request(
958 request_id,
884 - inputs,
959 + prompt,
960 sampling_params,
961 lora_request=lora_request,
962 trace_headers=trace_headers,
@@ -891,7 +966,7 @@ class AsyncLLMEngine:
966
967 async def encode(
968 self,
894 - inputs: PromptInputs,
969 + prompt: PromptType,
970 pooling_params: PoolingParams,
971 request_id: str,
972 lora_request: Optional[LoRARequest] = None,
@@ -904,8 +979,7 @@ class AsyncLLMEngine:
979 from the LLMEngine to the caller.
980
981 Args:
907 - inputs: The inputs to the LLM. See
908 - :class:`~vllm.inputs.PromptInputs`
982 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
983 for more details about the format of each input.
984 pooling_params: The pooling parameters of the request.
985 request_id: The unique id of the request.
@@ -959,7 +1033,7 @@ class AsyncLLMEngine:
1033 """
1034 async for output in await self.add_request(
1035 request_id,
962 - inputs,
1036 + prompt,
1037 pooling_params,
1038 lora_request=lora_request,
1039 trace_headers=trace_headers,
vllm/engine/llm_engine.py
+45 -7
@@ -6,7 +6,7 @@ from functools import partial
6 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7 Iterable, List, Mapping, NamedTuple, Optional)
8 from typing import Sequence as GenericSequence
9 -from typing import Set, Type, Union
9 +from typing import Set, Type, Union, overload
10
11 import torch
12 from typing_extensions import TypeVar
@@ -29,7 +29,7 @@ from vllm.executor.executor_base import ExecutorBase
29 from vllm.executor.gpu_executor import GPUExecutor
30 from vllm.executor.ray_utils import initialize_ray_cluster
31 from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32 - InputRegistry, LLMInputs, PromptInputs)
32 + InputRegistry, LLMInputs, PromptType)
33 from vllm.inputs.preprocess import InputPreprocessor
34 from vllm.logger import init_logger
35 from vllm.lora.request import LoRARequest
@@ -51,7 +51,7 @@ from vllm.transformers_utils.tokenizer_group import (
51 BaseTokenizerGroup, init_tokenizer_from_configs)
52 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
53 usage_message)
54 -from vllm.utils import Counter, Device, weak_bind
54 +from vllm.utils import Counter, Device, deprecate_kwargs, weak_bind
55 from vllm.version import __version__ as VLLM_VERSION
56
57 logger = init_logger(__name__)
@@ -689,16 +689,51 @@ class LLMEngine:
689 def stop_remote_worker_execution_loop(self) -> None:
690 self.model_executor.stop_remote_worker_execution_loop()
691
692 + @overload # DEPRECATED
693 def add_request(
694 self,
695 request_id: str,
695 - inputs: PromptInputs,
696 + *,
697 + inputs: PromptType,
698 params: Union[SamplingParams, PoolingParams],
699 arrival_time: Optional[float] = None,
700 lora_request: Optional[LoRARequest] = None,
701 trace_headers: Optional[Mapping[str, str]] = None,
702 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
703 priority: int = 0,
704 + ) -> None:
705 + ...
706 +
707 + @overload
708 + def add_request(
709 + self,
710 + request_id: str,
711 + prompt: PromptType,
712 + params: Union[SamplingParams, PoolingParams],
713 + arrival_time: Optional[float] = None,
714 + lora_request: Optional[LoRARequest] = None,
715 + trace_headers: Optional[Mapping[str, str]] = None,
716 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
717 + priority: int = 0,
718 + ) -> None:
719 + ...
720 +
721 + @deprecate_kwargs(
722 + "inputs",
723 + additional_message="Please use the 'prompt' parameter instead.",
724 + )
725 + def add_request(
726 + self,
727 + request_id: str,
728 + prompt: Optional[PromptType] = None,
729 + params: Optional[Union[SamplingParams, PoolingParams]] = None,
730 + arrival_time: Optional[float] = None,
731 + lora_request: Optional[LoRARequest] = None,
732 + trace_headers: Optional[Mapping[str, str]] = None,
733 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
734 + priority: int = 0,
735 + *,
736 + inputs: Optional[PromptType] = None, # DEPRECATED
737 ) -> None:
738 """Add a request to the engine's request pool.
739
@@ -708,8 +743,7 @@ class LLMEngine:
743
744 Args:
745 request_id: The unique ID of the request.
711 - inputs: The inputs to the LLM. See
712 - :class:`~vllm.inputs.PromptInputs`
746 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
747 for more details about the format of each input.
748 params: Parameters for sampling or pooling.
749 :class:`~vllm.SamplingParams` for text generation.
@@ -744,6 +778,10 @@ class LLMEngine:
778 >>> # continue the request processing
779 >>> ...
780 """
781 + if inputs is not None:
782 + prompt = inputs
783 + assert prompt is not None and params is not None
784 +
785 if lora_request is not None and not self.lora_config:
786 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
787 "not enabled!")
@@ -756,7 +794,7 @@ class LLMEngine:
794 arrival_time = time.time()
795
796 preprocessed_inputs = self.input_preprocessor.preprocess(
759 - inputs,
797 + prompt,
798 request_id=request_id,
799 lora_request=lora_request,
800 prompt_adapter_request=prompt_adapter_request,
vllm/engine/multiprocessing/__init__.py
+58 -3
@@ -1,13 +1,14 @@
1 from dataclasses import dataclass
2 from enum import Enum
3 -from typing import List, Mapping, Optional, Union
3 +from typing import List, Mapping, Optional, Union, overload
4
5 from vllm import PoolingParams
6 -from vllm.inputs import PromptInputs
6 +from vllm.inputs import PromptType
7 from vllm.lora.request import LoRARequest
8 from vllm.outputs import RequestOutput
9 from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.sampling_params import SamplingParams
11 +from vllm.utils import deprecate_kwargs
12
13 VLLM_RPC_SUCCESS_STR = "SUCCESS"
14
@@ -23,13 +24,67 @@ class MQEngineDeadError(RuntimeError):
24
25 @dataclass
26 class RPCProcessRequest:
26 - inputs: PromptInputs
27 + prompt: PromptType
28 params: Union[SamplingParams, PoolingParams]
29 request_id: str
30 lora_request: Optional[LoRARequest] = None
31 trace_headers: Optional[Mapping[str, str]] = None
32 prompt_adapter_request: Optional[PromptAdapterRequest] = None
33
34 + @overload # DEPRECATED
35 + def __init__(
36 + self,
37 + *,
38 + inputs: PromptType,
39 + params: Union[SamplingParams, PoolingParams],
40 + request_id: str,
41 + lora_request: Optional[LoRARequest] = None,
42 + trace_headers: Optional[Mapping[str, str]] = None,
43 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
44 + ) -> None:
45 + ...
46 +
47 + @overload
48 + def __init__(
49 + self,
50 + prompt: PromptType,
51 + params: Union[SamplingParams, PoolingParams],
52 + request_id: str,
53 + lora_request: Optional[LoRARequest] = None,
54 + trace_headers: Optional[Mapping[str, str]] = None,
55 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
56 + ) -> None:
57 + ...
58 +
59 + @deprecate_kwargs(
60 + "inputs",
61 + additional_message="Please use the 'prompt' parameter instead.",
62 + )
63 + def __init__(
64 + self,
65 + prompt: Optional[PromptType] = None,
66 + params: Optional[Union[SamplingParams, PoolingParams]] = None,
67 + request_id: Optional[str] = None,
68 + lora_request: Optional[LoRARequest] = None,
69 + trace_headers: Optional[Mapping[str, str]] = None,
70 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
71 + *,
72 + inputs: Optional[PromptType] = None, # DEPRECATED
73 + ) -> None:
74 + if inputs is not None:
75 + prompt = inputs
76 + assert (prompt is not None and params is not None
77 + and request_id is not None)
78 +
79 + super().__init__()
80 +
81 + self.prompt = prompt
82 + self.params = params
83 + self.request_id = request_id
84 + self.lora_request = lora_request
85 + self.trace_headers = trace_headers
86 + self.prompt_adapter_request = prompt_adapter_request
87 +
88
89 @dataclass
90 class RPCError:
vllm/engine/multiprocessing/client.py
+82 -13
@@ -3,7 +3,7 @@ import copy
3 import pickle
4 from contextlib import contextmanager, suppress
5 from typing import (Any, AsyncGenerator, Dict, Iterator, Mapping, Optional,
6 - Union)
6 + Union, overload)
7
8 import cloudpickle
9 import zmq
@@ -25,13 +25,14 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25 RPCUProfileRequest)
26 # yapf: enable
27 from vllm.envs import VLLM_RPC_TIMEOUT
28 -from vllm.inputs import PromptInputs
28 +from vllm.inputs import PromptType
29 from vllm.logger import init_logger
30 from vllm.lora.request import LoRARequest
31 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
32 from vllm.prompt_adapter.request import PromptAdapterRequest
33 from vllm.sampling_params import SamplingParams
34 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
35 +from vllm.utils import deprecate_kwargs
36
37 logger = init_logger(__name__)
38
@@ -367,14 +368,45 @@ class MQLLMEngineClient:
368 def dead_error(self) -> BaseException:
369 return ENGINE_DEAD_ERROR(self._errored_with)
370
371 + @overload # DEPRECATED
372 def generate(
373 self,
372 - inputs: PromptInputs,
374 + *,
375 + inputs: PromptType,
376 sampling_params: SamplingParams,
377 request_id: str,
378 lora_request: Optional[LoRARequest] = None,
379 trace_headers: Optional[Mapping[str, str]] = None,
377 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
380 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
381 + ) -> AsyncGenerator[RequestOutput, None]:
382 + ...
383 +
384 + @overload
385 + def generate(
386 + self,
387 + prompt: PromptType,
388 + sampling_params: SamplingParams,
389 + request_id: str,
390 + lora_request: Optional[LoRARequest] = None,
391 + trace_headers: Optional[Mapping[str, str]] = None,
392 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
393 + ) -> AsyncGenerator[RequestOutput, None]:
394 + ...
395 +
396 + @deprecate_kwargs(
397 + "inputs",
398 + additional_message="Please use the 'prompt' parameter instead.",
399 + )
400 + def generate(
401 + self,
402 + prompt: Optional[PromptType] = None,
403 + sampling_params: Optional[SamplingParams] = None,
404 + request_id: Optional[str] = None,
405 + lora_request: Optional[LoRARequest] = None,
406 + trace_headers: Optional[Mapping[str, str]] = None,
407 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
408 + *,
409 + inputs: Optional[PromptType] = None # DEPRECATED
410 ) -> AsyncGenerator[RequestOutput, None]:
411 """Generate outputs for a request.
412
@@ -383,8 +415,7 @@ class MQLLMEngineClient:
415 from the LLMEngine to the caller.
416
417 Args:
386 - inputs: The inputs to the LLM. See
387 - :class:`~vllm.inputs.PromptInputs`
418 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
419 for more details about the format of each input.
420 sampling_params: The sampling parameters of the request.
421 request_id: The unique id of the request.
@@ -393,17 +424,51 @@ class MQLLMEngineClient:
424 prompt_adapter_request: Prompt Adapter request to use
425 for generation, if any.
426 """
396 - return self._process_request(inputs, sampling_params, request_id,
427 + if inputs is not None:
428 + prompt = inputs
429 + assert (prompt is not None and sampling_params is not None
430 + and request_id is not None)
431 +
432 + return self._process_request(prompt, sampling_params, request_id,
433 lora_request, trace_headers,
434 prompt_adapter_request)
435
436 + @overload # DEPRECATED
437 def encode(
438 self,
402 - inputs: PromptInputs,
439 + *,
440 + inputs: PromptType,
441 pooling_params: PoolingParams,
442 request_id: str,
443 lora_request: Optional[LoRARequest] = None,
444 trace_headers: Optional[Mapping[str, str]] = None,
445 + ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
446 + ...
447 +
448 + @overload
449 + def encode(
450 + self,
451 + prompt: PromptType,
452 + pooling_params: PoolingParams,
453 + request_id: str,
454 + lora_request: Optional[LoRARequest] = None,
455 + trace_headers: Optional[Mapping[str, str]] = None,
456 + ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
457 + ...
458 +
459 + @deprecate_kwargs(
460 + "inputs",
461 + additional_message="Please use the 'prompt' parameter instead.",
462 + )
463 + def encode(
464 + self,
465 + prompt: Optional[PromptType] = None,
466 + pooling_params: Optional[PoolingParams] = None,
467 + request_id: Optional[str] = None,
468 + lora_request: Optional[LoRARequest] = None,
469 + trace_headers: Optional[Mapping[str, str]] = None,
470 + *,
471 + inputs: Optional[PromptType] = None # DEPRECATED
472 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
473 """Generate outputs for a request from an embedding model.
474
@@ -412,8 +477,7 @@ class MQLLMEngineClient:
477 from the LLMEngine to the caller.
478
479 Args:
415 - inputs: The inputs to the LLM. See
416 - :class:`~vllm.inputs.PromptInputs`
480 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
481 for more details about the format of each input.
482 pooling_params: The pooling parameters of the request.
483 request_id: The unique id of the request.
@@ -424,12 +488,17 @@ class MQLLMEngineClient:
488 The output `EmbeddingRequestOutput` objects from the LLMEngine
489 for the request.
490 """
427 - return self._process_request(inputs, pooling_params, request_id,
491 + if inputs is not None:
492 + prompt = inputs
493 + assert (prompt is not None and pooling_params is not None
494 + and request_id is not None)
495 +
496 + return self._process_request(prompt, pooling_params, request_id,
497 lora_request, trace_headers)
498
499 async def _process_request(
500 self,
432 - inputs: PromptInputs,
501 + prompt: PromptType,
502 params: Union[SamplingParams, PoolingParams],
503 request_id: str,
504 lora_request: Optional[LoRARequest] = None,
@@ -462,7 +531,7 @@ class MQLLMEngineClient:
531
532 request_bytes = pickle.dumps(
533 RPCProcessRequest(
465 - inputs=inputs,
534 + prompt=prompt,
535 params=params,
536 request_id=request_id,
537 lora_request=lora_request,
vllm/engine/multiprocessing/engine.py
+1 -1
@@ -278,7 +278,7 @@ class MQLLMEngine:
278 try:
279 self.engine.add_request(
280 request_id=request_id,
281 - inputs=request.inputs,
281 + prompt=request.prompt,
282 params=request.params,
283 lora_request=request.lora_request,
284 trace_headers=request.trace_headers,
vllm/engine/protocol.py
+4 -4
@@ -3,7 +3,7 @@ from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
3
4 from vllm.config import DecodingConfig, ModelConfig
5 from vllm.core.scheduler import SchedulerOutputs
6 -from vllm.inputs.data import PromptInputs
6 +from vllm.inputs.data import PromptType
7 from vllm.lora.request import LoRARequest
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -35,19 +35,19 @@ class EngineClient(Protocol):
35
36 def generate(
37 self,
38 - inputs: PromptInputs,
38 + prompt: PromptType,
39 sampling_params: SamplingParams,
40 request_id: str,
41 lora_request: Optional[LoRARequest] = None,
42 trace_headers: Optional[Mapping[str, str]] = None,
43 prompt_adapter_request: Optional[PromptAdapterRequest] = None
44 ) -> AsyncGenerator[RequestOutput, None]:
45 - """Generates outputs for a request"""
45 + """Generate outputs for a request."""
46 ...
47
48 def encode(
49 self,
50 - inputs: PromptInputs,
50 + prompt: PromptType,
51 pooling_params: PoolingParams,
52 request_id: str,
53 lora_request: Optional[LoRARequest] = None,
vllm/entrypoints/llm.py
+35 -33
@@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
12 apply_hf_chat_template,
13 apply_mistral_chat_template,
14 parse_chat_messages)
15 -from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
15 +from vllm.inputs import PromptType, TextPrompt, TokensPrompt
16 from vllm.inputs.parse import parse_and_batch_prompt
17 from vllm.logger import init_logger
18 from vllm.lora.request import LoRARequest
@@ -293,8 +293,8 @@ class LLM:
293 @overload
294 def generate(
295 self,
296 - inputs: Union[PromptInputs, Sequence[PromptInputs]],
297 - /, # We may enable `inputs` keyword after removing the old API
296 + prompts: Union[PromptType, Sequence[PromptType]],
297 + /,
298 *,
299 sampling_params: Optional[Union[SamplingParams,
300 Sequence[SamplingParams]]] = None,
@@ -304,14 +304,13 @@ class LLM:
304 ...
305
306 @deprecate_kwargs(
307 - "prompts",
307 "prompt_token_ids",
308 is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
310 - additional_message="Please use the 'inputs' parameter instead.",
309 + additional_message="Please use the 'prompts' parameter instead.",
310 )
311 def generate(
312 self,
314 - prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
313 + prompts: Union[Union[PromptType, Sequence[PromptType]],
314 Optional[Union[str, List[str]]]] = None,
315 sampling_params: Optional[Union[SamplingParams,
316 Sequence[SamplingParams]]] = None,
@@ -330,7 +329,9 @@ class LLM:
329 into a single list and pass it to this method.
330
331 Args:
333 - inputs: A list of inputs to generate completions for.
332 + prompts: The prompts to the LLM. You may pass a sequence of prompts
333 + for batch inference. See :class:`~vllm.inputs.PromptType`
334 + for more details about the format of each prompts.
335 sampling_params: The sampling parameters for text generation. If
336 None, we use the default sampling parameters.
337 When it is a single value, it is applied to every prompt.
@@ -358,12 +359,13 @@ class LLM:
359 "models (XForCausalLM, XForConditionalGeneration).")
360
361 if prompt_token_ids is not None:
361 - inputs = self._convert_v1_inputs(
362 + parsed_prompts = self._convert_v1_inputs(
363 prompts=cast(Optional[Union[str, List[str]]], prompts),
364 prompt_token_ids=prompt_token_ids,
365 )
366 else:
366 - inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
367 + parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
368 + prompts)
369
370 if isinstance(guided_options_request, dict):
371 if len(guided_options_request) > 1:
@@ -378,7 +380,7 @@ class LLM:
380 sampling_params = SamplingParams()
381
382 self._validate_and_add_requests(
381 - inputs=inputs,
383 + prompts=parsed_prompts,
384 params=sampling_params,
385 lora_request=lora_request,
386 prompt_adapter_request=prompt_adapter_request,
@@ -648,8 +650,8 @@ class LLM:
650 @overload
651 def encode(
652 self,
651 - inputs: Union[PromptInputs, Sequence[PromptInputs]],
652 - /, # We may enable `inputs` keyword after removing the old API
653 + prompts: Union[PromptType, Sequence[PromptType]],
654 + /,
655 *,
656 pooling_params: Optional[Union[PoolingParams,
657 Sequence[PoolingParams]]] = None,
@@ -659,14 +661,13 @@ class LLM:
661 ...
662
663 @deprecate_kwargs(
662 - "prompts",
664 "prompt_token_ids",
665 is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
665 - additional_message="Please use the 'inputs' parameter instead.",
666 + additional_message="Please use the 'prompts' parameter instead.",
667 )
668 def encode(
669 self,
669 - prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
670 + prompts: Union[Union[PromptType, Sequence[PromptType]],
671 Optional[Union[str, List[str]]]] = None,
672 pooling_params: Optional[Union[PoolingParams,
673 Sequence[PoolingParams]]] = None,
@@ -682,9 +683,9 @@ class LLM:
683 into a single list and pass it to this method.
684
685 Args:
685 - inputs: The inputs to the LLM. You may pass a sequence of inputs for
686 - batch inference. See :class:`~vllm.inputs.PromptInputs`
687 - for more details about the format of each input.
686 + prompts: The prompts to the LLM. You may pass a sequence of prompts
687 + for batch inference. See :class:`~vllm.inputs.PromptType`
688 + for more details about the format of each prompts.
689 pooling_params: The pooling parameters for pooling. If None, we
690 use the default pooling parameters.
691 use_tqdm: Whether to use tqdm to display the progress bar.
@@ -707,19 +708,20 @@ class LLM:
708 )
709
710 if prompt_token_ids is not None:
710 - inputs = self._convert_v1_inputs(
711 + parsed_prompts = self._convert_v1_inputs(
712 prompts=cast(Optional[Union[str, List[str]]], prompts),
713 prompt_token_ids=prompt_token_ids,
714 )
715 else:
715 - inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
716 + parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
717 + prompts)
718
719 if pooling_params is None:
720 # Use default pooling params.
721 pooling_params = PoolingParams()
722
723 self._validate_and_add_requests(
722 - inputs=inputs,
724 + prompts=parsed_prompts,
725 params=pooling_params,
726 lora_request=lora_request,
727 prompt_adapter_request=prompt_adapter_request,
@@ -763,9 +765,9 @@ class LLM:
765 raise ValueError("Either prompts or prompt_token_ids must be "
766 "provided.")
767
766 - inputs: List[PromptInputs] = []
768 + parsed_prompts: List[PromptType] = []
769 for i in range(num_requests):
768 - item: PromptInputs
770 + item: PromptType
771
772 if prompts is not None:
773 item = TextPrompt(prompt=prompts[i])
@@ -774,13 +776,13 @@ class LLM:
776 else:
777 raise AssertionError
778
777 - inputs.append(item)
779 + parsed_prompts.append(item)
780
779 - return inputs
781 + return parsed_prompts
782
783 def _validate_and_add_requests(
784 self,
783 - inputs: Union[PromptInputs, Sequence[PromptInputs]],
785 + prompts: Union[PromptType, Sequence[PromptType]],
786 params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
787 Sequence[PoolingParams]],
788 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
@@ -788,11 +790,11 @@ class LLM:
790 guided_options: Optional[GuidedDecodingRequest] = None,
791 priority: Optional[List[int]] = None,
792 ) -> None:
791 - if isinstance(inputs, (str, dict)):
793 + if isinstance(prompts, (str, dict)):
794 # Convert a single prompt to a list.
793 - inputs = [inputs]
795 + prompts = [prompts]
796
795 - num_requests = len(inputs)
797 + num_requests = len(prompts)
798 if isinstance(params, list) and len(params) != num_requests:
799 raise ValueError("The lengths of prompts and params "
800 "must be the same.")
@@ -809,9 +811,9 @@ class LLM:
811 sp.output_kind = RequestOutputKind.FINAL_ONLY
812
813 # Add requests to the engine.
812 - for i, request_inputs in enumerate(inputs):
814 + for i, prompt in enumerate(prompts):
815 self._add_request(
814 - request_inputs,
816 + prompt,
817 params[i] if isinstance(params, Sequence) else params,
818 lora_request=lora_request[i] if isinstance(
819 lora_request, Sequence) else lora_request,
@@ -821,7 +823,7 @@ class LLM:
823
824 def _add_request(
825 self,
824 - inputs: PromptInputs,
826 + prompt: PromptType,
827 params: Union[SamplingParams, PoolingParams],
828 lora_request: Optional[LoRARequest] = None,
829 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -830,7 +832,7 @@ class LLM:
832 request_id = str(next(self.request_counter))
833 self.llm_engine.add_request(
834 request_id,
833 - inputs,
835 + prompt,
836 params,
837 lora_request=lora_request,
838 prompt_adapter_request=prompt_adapter_request,
vllm/inputs/__init__.py
+17 -3
@@ -1,5 +1,5 @@
1 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2 - LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
2 + LLMInputs, PromptType, SingletonPrompt, TextPrompt,
3 TokensPrompt, build_explicit_enc_dec_prompt,
4 to_enc_dec_tuple_list, zip_enc_dec_prompts)
5 from .registry import InputContext, InputRegistry
@@ -16,8 +16,8 @@ See also:
16 __all__ = [
17 "TextPrompt",
18 "TokensPrompt",
19 - "PromptInputs",
20 - "SingletonPromptInputs",
19 + "PromptType",
20 + "SingletonPrompt",
21 "ExplicitEncoderDecoderPrompt",
22 "LLMInputs",
23 "EncoderDecoderLLMInputs",
@@ -28,3 +28,17 @@ __all__ = [
28 "InputContext",
29 "InputRegistry",
30 ]
31 +
32 +
33 +def __getattr__(name: str):
34 + if name == "PromptInput":
35 + import warnings
36 +
37 + msg = ("PromptInput has been renamed to PromptType. "
38 + "The original name will be removed in an upcoming version.")
39 +
40 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
41 +
42 + return PromptType
43 +
44 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/data.py
+31 -22
@@ -33,7 +33,7 @@ class TokensPrompt(TypedDict):
33 """
34
35
36 -SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
36 +SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
37 """
38 Set of possible schemas for a single LLM input:
39
@@ -46,7 +46,7 @@ which may be utilized for encoder/decoder models when
46 the user desires to express both the encoder & decoder
47 prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
49 -A prompt of type :class:`SingletonPromptInputs` may be employed
49 +A prompt of type :class:`SingletonPrompt` may be employed
50 as (1) input to a decoder-only model, (2) input to
51 the encoder of an encoder/decoder model, in the scenario
52 where the decoder-prompt is not specified explicitly, or
@@ -55,33 +55,32 @@ more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55 """
56
57 _T1_co = TypeVar("_T1_co",
58 - bound=SingletonPromptInputs,
59 - default=SingletonPromptInputs,
58 + bound=SingletonPrompt,
59 + default=SingletonPrompt,
60 covariant=True)
61 _T2_co = TypeVar("_T2_co",
62 - bound=SingletonPromptInputs,
63 - default=SingletonPromptInputs,
62 + bound=SingletonPrompt,
63 + default=SingletonPrompt,
64 covariant=True)
65
66
67 # TODO: Make fields ReadOnly once mypy supports it
68 class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
69 - """Represents an encoder/decoder model input prompt,
70 - comprising an explicit encoder prompt and a
71 - decoder prompt.
69 + """
70 + Represents an encoder/decoder model input prompt,
71 + comprising an explicit encoder prompt and a decoder prompt.
72
73 - The encoder and decoder prompts, respectively,
74 - may formatted according to any of the
75 - :class:`SingletonPromptInputs` schemas, and are not
76 - required to have the same schema.
73 + The encoder and decoder prompts, respectively, may be formatted
74 + according to any of the :class:`SingletonPrompt` schemas,
75 + and are not required to have the same schema.
76
77 Only the encoder prompt may have multi-modal data.
78
79 Note that an :class:`ExplicitEncoderDecoderPrompt` may not
80 be used as an input to a decoder-only model,
82 - and that the `encoder_prompt` and `decoder_prompt`
81 + and that the :code:`encoder_prompt` and :code:`decoder_prompt`
82 fields of this data structure themselves must be
84 - :class:`SingletonPromptInputs` instances.
83 + :class:`SingletonPrompt` instances.
84 """
85
86 encoder_prompt: _T1_co
@@ -89,7 +88,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
88 decoder_prompt: Optional[_T2_co]
89
90
92 -PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
91 +PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
92 """
93 Set of possible schemas for an LLM input, including
94 both decoder-only and encoder/decoder input types:
@@ -146,12 +145,8 @@ class EncoderDecoderLLMInputs(LLMInputs):
145 """
146
147
149 -_T1 = TypeVar("_T1",
150 - bound=SingletonPromptInputs,
151 - default=SingletonPromptInputs)
152 -_T2 = TypeVar("_T2",
153 - bound=SingletonPromptInputs,
154 - default=SingletonPromptInputs)
148 +_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
149 +_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
150
151
152 def build_explicit_enc_dec_prompt(
@@ -182,3 +177,17 @@ def to_enc_dec_tuple_list(
177 return [(enc_dec_prompt["encoder_prompt"],
178 enc_dec_prompt["decoder_prompt"])
179 for enc_dec_prompt in enc_dec_prompts]
180 +
181 +
182 +def __getattr__(name: str):
183 + if name == "PromptInput":
184 + import warnings
185 +
186 + msg = ("PromptInput has been renamed to PromptType. "
187 + "The original name will be removed in an upcoming version.")
188 +
189 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
190 +
191 + return PromptType
192 +
193 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/parse.py
+11 -11
@@ -5,7 +5,7 @@ from typing_extensions import TypeIs
5 from vllm.utils import is_list_of
6
7 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8 - LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
8 + LLMInputs, PromptType, SingletonPrompt, TextPrompt,
9 TokensPrompt)
10
11
@@ -81,23 +81,23 @@ class ParsedTokensPrompt(TypedDict):
81
82
83 def parse_singleton_prompt(
84 - inputs: SingletonPromptInputs,
84 + prompt: SingletonPrompt,
85 ) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86 - if isinstance(inputs, str):
87 - return ParsedStrPrompt(type="str", content=inputs)
88 - elif isinstance(inputs, dict):
89 - if "prompt_token_ids" in inputs:
86 + if isinstance(prompt, str):
87 + return ParsedStrPrompt(type="str", content=prompt)
88 + elif isinstance(prompt, dict):
89 + if "prompt_token_ids" in prompt:
90 return ParsedTokensPrompt(type="tokens",
91 - content=inputs) # type: ignore
92 - elif "prompt" in inputs:
93 - return ParsedTextPrompt(type="text", content=inputs)
91 + content=prompt) # type: ignore
92 + elif "prompt" in prompt:
93 + return ParsedTextPrompt(type="text", content=prompt)
94
95 raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98 def is_explicit_encoder_decoder_prompt(
99 - inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 - return isinstance(inputs, dict) and "encoder_prompt" in inputs
99 + prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 + return isinstance(prompt, dict) and "encoder_prompt" in prompt
101
102
103 def is_valid_encoder_decoder_llm_inputs(
vllm/inputs/preprocess.py
+43 -43
@@ -10,8 +10,8 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11 from vllm.utils import print_warning_once
12
13 -from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
14 - SingletonPromptInputs)
13 +from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
14 + SingletonPrompt)
15 from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
16
17 if TYPE_CHECKING:
@@ -209,7 +209,7 @@ class InputPreprocessor:
209
210 def _extract_prompt_components(
211 self,
212 - inputs: SingletonPromptInputs,
212 + prompt: SingletonPrompt,
213 request_id: str,
214 lora_request: Optional[LoRARequest] = None,
215 ) -> PromptComponents:
@@ -219,7 +219,7 @@ class InputPreprocessor:
219 Arguments:
220
221 * request_id
222 - * inputs: single encoder or decoder input prompt
222 + * prompt: single encoder or decoder input prompt
223 * lora_request: this is only valid for decoder prompts
224
225 Returns:
@@ -229,24 +229,24 @@ class InputPreprocessor:
229 * multi_modal_data
230 '''
231
232 - parsed = parse_singleton_prompt(inputs)
232 + parsed = parse_singleton_prompt(prompt)
233
234 if parsed["type"] == "str":
235 - prompt = parsed["content"]
235 + prompt_text = parsed["content"]
236 prompt_token_ids = self._tokenize_prompt(
237 - prompt,
237 + prompt_text,
238 request_id=request_id,
239 lora_request=lora_request,
240 )
241 multi_modal_data = None
242 elif parsed["type"] == "tokens":
243 - prompt = None
243 + prompt_text = None
244 prompt_token_ids = parsed["content"]["prompt_token_ids"]
245 multi_modal_data = parsed["content"].get("multi_modal_data")
246 elif parsed["type"] == "text":
247 - prompt = parsed["content"]["prompt"]
247 + prompt_text = parsed["content"]["prompt"]
248 prompt_token_ids = self._tokenize_prompt(
249 - prompt,
249 + prompt_text,
250 request_id=request_id,
251 lora_request=lora_request,
252 )
@@ -254,33 +254,33 @@ class InputPreprocessor:
254 else:
255 assert_never(parsed)
256
257 - return prompt, prompt_token_ids, multi_modal_data
257 + return prompt_text, prompt_token_ids, multi_modal_data
258
259 async def _extract_prompt_components_async(
260 self,
261 - inputs: SingletonPromptInputs,
261 + prompt: SingletonPrompt,
262 request_id: str,
263 lora_request: Optional[LoRARequest] = None,
264 ) -> PromptComponents:
265 """Async version of :meth:`_extract_prompt_components`."""
266 - parsed = parse_singleton_prompt(inputs)
266 + parsed = parse_singleton_prompt(prompt)
267
268 if parsed["type"] == "str":
269 - prompt = parsed["content"]
269 + prompt_text = parsed["content"]
270 prompt_token_ids = await self._tokenize_prompt_async(
271 - prompt,
271 + prompt_text,
272 request_id=request_id,
273 lora_request=lora_request,
274 )
275 multi_modal_data = None
276 elif parsed["type"] == "tokens":
277 - prompt = None
277 + prompt_text = None
278 prompt_token_ids = parsed["content"]["prompt_token_ids"]
279 multi_modal_data = parsed["content"].get("multi_modal_data")
280 elif parsed["type"] == "text":
281 - prompt = parsed["content"]["prompt"]
281 + prompt_text = parsed["content"]["prompt"]
282 prompt_token_ids = await self._tokenize_prompt_async(
283 - prompt,
283 + prompt_text,
284 request_id=request_id,
285 lora_request=lora_request,
286 )
@@ -288,7 +288,7 @@ class InputPreprocessor:
288 else:
289 assert_never(parsed)
290
291 - return prompt, prompt_token_ids, multi_modal_data
291 + return prompt_text, prompt_token_ids, multi_modal_data
292
293 def _build_enc_dec_llm_inputs(
294 self,
@@ -321,7 +321,7 @@ class InputPreprocessor:
321
322 def _process_encoder_decoder_prompt(
323 self,
324 - inputs: PromptInputs,
324 + prompt: PromptType,
325 request_id: str,
326 ) -> EncoderDecoderLLMInputs:
327 '''
@@ -349,7 +349,7 @@ class InputPreprocessor:
349
350 Arguments:
351
352 - * inputs: an input prompt
352 + * prompt: an input prompt
353 * request_id
354
355 Returns:
@@ -360,13 +360,13 @@ class InputPreprocessor:
360 encoder_comps: PromptComponents
361 decoder_comps: DecoderPromptComponents
362
363 - if is_explicit_encoder_decoder_prompt(inputs):
363 + if is_explicit_encoder_decoder_prompt(prompt):
364 encoder_comps = self._extract_prompt_components(
365 - inputs["encoder_prompt"],
365 + prompt["encoder_prompt"],
366 request_id=request_id,
367 )
368
369 - if (decoder_input := inputs["decoder_prompt"]) is None:
369 + if (decoder_input := prompt["decoder_prompt"]) is None:
370 decoder_comps = None, None, None
371 else:
372 decoder_comps = self._extract_prompt_components(
@@ -375,7 +375,7 @@ class InputPreprocessor:
375 )
376 else:
377 encoder_comps = self._extract_prompt_components(
378 - inputs,
378 + prompt,
379 request_id=request_id,
380 )
381
@@ -385,20 +385,20 @@ class InputPreprocessor:
385
386 async def _process_encoder_decoder_prompt_async(
387 self,
388 - inputs: PromptInputs,
388 + prompt: PromptType,
389 request_id: str,
390 ) -> EncoderDecoderLLMInputs:
391 """Async version of :meth:`_process_encoder_decoder_prompt`."""
392 encoder_comps: PromptComponents
393 decoder_comps: DecoderPromptComponents
394
395 - if is_explicit_encoder_decoder_prompt(inputs):
395 + if is_explicit_encoder_decoder_prompt(prompt):
396 encoder_task = self._extract_prompt_components_async(
397 - inputs["encoder_prompt"],
397 + prompt["encoder_prompt"],
398 request_id=request_id,
399 )
400
401 - if (decoder_input := inputs["decoder_prompt"]) is None:
401 + if (decoder_input := prompt["decoder_prompt"]) is None:
402 encoder_comps = await encoder_task
403 decoder_comps = None, None, None
404 else:
@@ -411,7 +411,7 @@ class InputPreprocessor:
411 encoder_task, decoder_task)
412 else:
413 encoder_comps = await self._extract_prompt_components_async(
414 - inputs,
414 + prompt,
415 request_id=request_id,
416 )
417
@@ -435,7 +435,7 @@ class InputPreprocessor:
435
436 def _process_decoder_only_prompt(
437 self,
438 - inputs: SingletonPromptInputs,
438 + prompt: SingletonPrompt,
439 request_id: str,
440 lora_request: Optional[LoRARequest] = None,
441 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -446,7 +446,7 @@ class InputPreprocessor:
446
447 Arguments:
448
449 - * inputs: input prompt
449 + * prompt: input prompt
450 * request_id
451 * lora_request
452 * prompt_adapter_request
@@ -457,7 +457,7 @@ class InputPreprocessor:
457 '''
458
459 prompt_comps = self._extract_prompt_components(
460 - inputs,
460 + prompt,
461 request_id=request_id,
462 lora_request=lora_request,
463 )
@@ -469,14 +469,14 @@ class InputPreprocessor:
469
470 async def _process_decoder_only_prompt_async(
471 self,
472 - inputs: SingletonPromptInputs,
472 + prompt: SingletonPrompt,
473 request_id: str,
474 lora_request: Optional[LoRARequest] = None,
475 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
476 ) -> LLMInputs:
477 """Async version of :meth:`_process_decoder_only_prompt`."""
478 prompt_comps = await self._extract_prompt_components_async(
479 - inputs,
479 + prompt,
480 request_id=request_id,
481 lora_request=lora_request,
482 )
@@ -488,7 +488,7 @@ class InputPreprocessor:
488
489 def preprocess(
490 self,
491 - inputs: PromptInputs,
491 + prompt: PromptType,
492 request_id: str,
493 lora_request: Optional[LoRARequest] = None,
494 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -498,17 +498,17 @@ class InputPreprocessor:
498 # Encoder-decoder model requires special mapping of
499 # input prompts to encoder & decoder
500 return self._process_encoder_decoder_prompt(
501 - inputs,
501 + prompt,
502 request_id=request_id,
503 )
504
505 - if is_explicit_encoder_decoder_prompt(inputs):
505 + if is_explicit_encoder_decoder_prompt(prompt):
506 raise ValueError("Cannot pass encoder-decoder prompt "
507 "to decoder-only models")
508
509 # Decoder-only operation
510 return self._process_decoder_only_prompt(
511 - inputs,
511 + prompt,
512 request_id=request_id,
513 lora_request=lora_request,
514 prompt_adapter_request=prompt_adapter_request,
@@ -516,7 +516,7 @@ class InputPreprocessor:
516
517 async def preprocess_async(
518 self,
519 - inputs: PromptInputs,
519 + prompt: PromptType,
520 request_id: str,
521 lora_request: Optional[LoRARequest] = None,
522 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -526,17 +526,17 @@ class InputPreprocessor:
526 # Encoder-decoder model requires special mapping of
527 # input prompts to encoder & decoder
528 return await self._process_encoder_decoder_prompt_async(
529 - inputs,
529 + prompt,
530 request_id=request_id,
531 )
532
533 - if is_explicit_encoder_decoder_prompt(inputs):
533 + if is_explicit_encoder_decoder_prompt(prompt):
534 raise ValueError("Cannot pass encoder-decoder prompt "
535 "to decoder-only models")
536
537 # Decoder-only operation
538 return await self._process_decoder_only_prompt_async(
539 - inputs,
539 + prompt,
540 request_id=request_id,
541 lora_request=lora_request,
542 prompt_adapter_request=prompt_adapter_request,