@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4f1ba0844b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

Revert "rename PromptInputs and inputs with backward compatibility (#8760) (#8810)

Simon Mo committed Sep 25, 2024 at 10:36 UTC 4f1ba0844b83b4e7d0ff1672b7ba502ce8732f95
21 files changed +245 -438
benchmarks/benchmark_latency.py
+4 -4
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
13 from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14 -from vllm.inputs import PromptType
14 +from vllm.inputs import PromptInputs
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61 dummy_prompt_token_ids = np.random.randint(10000,
62 size=(args.batch_size,
63 args.input_len))
64 - dummy_prompts: List[PromptType] = [{
64 + dummy_inputs: List[PromptInputs] = [{
65 "prompt_token_ids": batch
66 } for batch in dummy_prompt_token_ids.tolist()]
67
@@ -74,13 +74,13 @@ def main(args: argparse.Namespace):
74 ],
75 on_trace_ready=torch.profiler.tensorboard_trace_handler(
76 str(profile_dir))) as p:
77 - llm.generate(dummy_prompts,
77 + llm.generate(dummy_inputs,
78 sampling_params=sampling_params,
79 use_tqdm=False)
80 print(p.key_averages())
81 else:
82 start_time = time.perf_counter()
83 - llm.generate(dummy_prompts,
83 + llm.generate(dummy_inputs,
84 sampling_params=sampling_params,
85 use_tqdm=False)
86 end_time = time.perf_counter()
docs/source/dev/multimodal/multimodal_index.rst
+1 -1
@@ -8,7 +8,7 @@ Multi-Modality
8 vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10 Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11 -via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
11 +via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
12
13 Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14 by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1 -1
@@ -1,7 +1,7 @@
1 LLM Inputs
2 ==========
3
4 -.. autodata:: vllm.inputs.PromptType
4 +.. autodata:: vllm.inputs.PromptInputs
5
6 .. autoclass:: vllm.inputs.TextPrompt
7 :show-inheritance:
docs/source/models/vlm.rst
+1 -1
@@ -27,7 +27,7 @@ The :class:`~vllm.LLM` class can be instantiated in much the same way as languag
27 We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28 the above snippet. Specifically, ``image_feature_size`` can no longer be specified as we now calculate that internally for each model.
29
30 -To pass an image to the model, note the following in :class:`vllm.inputs.PromptType`:
30 +To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
31
32 * ``prompt``: The prompt should follow the format that is documented on HuggingFace.
33 * ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/async_engine/test_async_llm_engine.py
+3 -5
@@ -86,19 +86,17 @@ class MockAsyncLLMEngine(AsyncLLMEngine):
86
87 @pytest.mark.asyncio
88 async def test_new_requests_event():
89 - params = SamplingParams()
90 -
89 engine = MockAsyncLLMEngine()
90 engine.start_background_loop()
91 await asyncio.sleep(0.01)
92 assert engine.engine.step_calls == 0
93
96 - await engine.add_request("1", "", params)
94 + await engine.add_request("1", "", None)
95 await asyncio.sleep(0.01)
96 assert engine.engine.add_request_calls == 1
97 assert engine.engine.step_calls == 1
98
101 - await engine.add_request("2", "", params)
99 + await engine.add_request("2", "", None)
100 engine.engine.generate("2")
101 await asyncio.sleep(0)
102 await asyncio.sleep(0)
@@ -113,7 +111,7 @@ async def test_new_requests_event():
111 await asyncio.sleep(0.001)
112 assert engine.engine.step_calls == old_step_calls
113
116 - await engine.add_request("3", "", params)
114 + await engine.add_request("3", "", None)
115 await asyncio.sleep(0.01)
116 assert engine.engine.add_request_calls == 3
117 assert engine.engine.step_calls == old_step_calls + 1
tests/entrypoints/llm/test_encode.py
+34
@@ -49,6 +49,21 @@ def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
49 assert [o.outputs for o in o1] == [o.outputs for o in o2]
50
51
52 +@pytest.mark.skip_global_cleanup
53 +@pytest.mark.parametrize('prompt', PROMPTS)
54 +def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
55 + pooling_params = PoolingParams()
56 +
57 + with pytest.warns(DeprecationWarning, match="'prompts'"):
58 + v1_output = llm.encode(prompts=prompt, pooling_params=pooling_params)
59 +
60 + v2_output = llm.encode(prompt, pooling_params=pooling_params)
61 + assert_outputs_equal(v1_output, v2_output)
62 +
63 + v2_output = llm.encode({"prompt": prompt}, pooling_params=pooling_params)
64 + assert_outputs_equal(v1_output, v2_output)
65 +
66 +
67 @pytest.mark.skip_global_cleanup
68 @pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
69 def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -64,6 +79,25 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
79 assert_outputs_equal(v1_output, v2_output)
80
81
82 +@pytest.mark.skip_global_cleanup
83 +def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84 + pooling_params = PoolingParams()
85 +
86 + with pytest.warns(DeprecationWarning, match="'prompts'"):
87 + v1_output = llm.encode(prompts=PROMPTS, pooling_params=pooling_params)
88 +
89 + v2_output = llm.encode(PROMPTS, pooling_params=pooling_params)
90 + assert_outputs_equal(v1_output, v2_output)
91 +
92 + v2_output = llm.encode(
93 + [{
94 + "prompt": p
95 + } for p in PROMPTS],
96 + pooling_params=pooling_params,
97 + )
98 + assert_outputs_equal(v1_output, v2_output)
99 +
100 +
101 @pytest.mark.skip_global_cleanup
102 def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
103 pooling_params = PoolingParams()
tests/entrypoints/llm/test_generate.py
+37
@@ -47,6 +47,23 @@ def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
47 assert [o.outputs for o in o1] == [o.outputs for o in o2]
48
49
50 +@pytest.mark.skip_global_cleanup
51 +@pytest.mark.parametrize('prompt', PROMPTS)
52 +def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
53 + sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
54 +
55 + with pytest.warns(DeprecationWarning, match="'prompts'"):
56 + v1_output = llm.generate(prompts=prompt,
57 + sampling_params=sampling_params)
58 +
59 + v2_output = llm.generate(prompt, sampling_params=sampling_params)
60 + assert_outputs_equal(v1_output, v2_output)
61 +
62 + v2_output = llm.generate({"prompt": prompt},
63 + sampling_params=sampling_params)
64 + assert_outputs_equal(v1_output, v2_output)
65 +
66 +
67 @pytest.mark.skip_global_cleanup
68 @pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
69 def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
@@ -62,6 +79,26 @@ def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
79 assert_outputs_equal(v1_output, v2_output)
80
81
82 +@pytest.mark.skip_global_cleanup
83 +def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
84 + sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
85 +
86 + with pytest.warns(DeprecationWarning, match="'prompts'"):
87 + v1_output = llm.generate(prompts=PROMPTS,
88 + sampling_params=sampling_params)
89 +
90 + v2_output = llm.generate(PROMPTS, sampling_params=sampling_params)
91 + assert_outputs_equal(v1_output, v2_output)
92 +
93 + v2_output = llm.generate(
94 + [{
95 + "prompt": p
96 + } for p in PROMPTS],
97 + sampling_params=sampling_params,
98 + )
99 + assert_outputs_equal(v1_output, v2_output)
100 +
101 +
102 @pytest.mark.skip_global_cleanup
103 def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
104 sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
tests/mq_llm_engine/test_error_handling.py
+6 -6
@@ -61,7 +61,7 @@ async def test_evil_forward(tmp_socket):
61
62 # Throws an error in first forward pass.
63 with pytest.raises(RAISED_ERROR):
64 - async for _ in client.generate(prompt="Hello my name is",
64 + async for _ in client.generate(inputs="Hello my name is",
65 sampling_params=SamplingParams(),
66 request_id=uuid.uuid4()):
67 pass
@@ -69,7 +69,7 @@ async def test_evil_forward(tmp_socket):
69
70 # Engine is errored, should get ENGINE_DEAD_ERROR.
71 with pytest.raises(MQEngineDeadError):
72 - async for _ in client.generate(prompt="Hello my name is",
72 + async for _ in client.generate(inputs="Hello my name is",
73 sampling_params=SamplingParams(),
74 request_id=uuid.uuid4()):
75 pass
@@ -118,7 +118,7 @@ async def test_failed_health_check(tmp_socket):
118
119 # Generate call should throw ENGINE_DEAD_ERROR
120 with pytest.raises(MQEngineDeadError):
121 - async for _ in client.generate(prompt="Hello my name is",
121 + async for _ in client.generate(inputs="Hello my name is",
122 sampling_params=SamplingParams(),
123 request_id=uuid.uuid4()):
124 pass
@@ -160,7 +160,7 @@ async def test_failed_abort(tmp_socket):
160 # with reference to the original KeyError("foo")
161 with pytest.raises(MQEngineDeadError) as execinfo:
162 async for _ in client.generate(
163 - prompt="Hello my name is",
163 + inputs="Hello my name is",
164 sampling_params=SamplingParams(max_tokens=10),
165 request_id=uuid.uuid4()):
166 pass
@@ -183,7 +183,7 @@ async def test_bad_request(tmp_socket):
183
184 # Invalid request should fail, but not crash the server.
185 with pytest.raises(ValueError):
186 - async for _ in client.generate(prompt="Hello my name is",
186 + async for _ in client.generate(inputs="Hello my name is",
187 sampling_params=SamplingParams(),
188 request_id="abcd-1",
189 lora_request=LoRARequest(
@@ -192,7 +192,7 @@ async def test_bad_request(tmp_socket):
192 pass
193
194 # This request should be okay.
195 - async for _ in client.generate(prompt="Hello my name is",
195 + async for _ in client.generate(inputs="Hello my name is",
196 sampling_params=SamplingParams(),
197 request_id="abcd-2"):
198 pass
tests/mq_llm_engine/utils.py
+1 -1
@@ -20,7 +20,7 @@ async def generate(
20 count = 0
21 async for out in client.generate(
22 request_id=request_id,
23 - prompt="Hello my name is Robert and",
23 + inputs="Hello my name is Robert and",
24 sampling_params=SamplingParams(max_tokens=num_tokens,
25 temperature=0)):
26
vllm/__init__.py
+2 -2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 -from vllm.inputs import PromptType, TextPrompt, TokensPrompt
8 +from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11 EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19 "__version_tuple__",
20 "LLM",
21 "ModelRegistry",
22 - "PromptType",
22 + "PromptInputs",
23 "TextPrompt",
24 "TokensPrompt",
25 "SamplingParams",
vllm/engine/async_llm_engine.py
+18 -92
@@ -2,8 +2,8 @@ import asyncio
2 import time
3 import weakref
4 from functools import partial
5 -from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
6 - List, Mapping, Optional, Set, Tuple, Type, Union, overload)
5 +from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6 + Mapping, Optional, Set, Tuple, Type, Union)
7 from weakref import ReferenceType
8
9 import vllm.envs as envs
@@ -17,7 +17,7 @@ from vllm.engine.metrics_types import StatLoggerBase
17 from vllm.executor.executor_base import ExecutorAsyncBase
18 from vllm.executor.gpu_executor import GPUExecutorAsync
19 from vllm.executor.ray_utils import initialize_ray_cluster
20 -from vllm.inputs import PromptType
20 +from vllm.inputs import PromptInputs
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -28,7 +28,7 @@ from vllm.sampling_params import SamplingParams
28 from vllm.sequence import ExecuteModelRequest
29 from vllm.transformers_utils.tokenizer import AnyTokenizer
30 from vllm.usage.usage_lib import UsageContext
31 -from vllm.utils import deprecate_kwargs, weak_bind
31 +from vllm.utils import weak_bind
32
33 logger = init_logger(__name__)
34 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -402,54 +402,17 @@ class _AsyncLLMEngine(LLMEngine):
402 """Stop the remote worker execution loop."""
403 await self.model_executor.stop_remote_worker_execution_loop_async()
404
405 - @overload # DEPRECATED
405 async def add_request_async(
406 self,
407 request_id: str,
409 - *,
410 - inputs: PromptType,
408 + inputs: PromptInputs,
409 params: Union[SamplingParams, PoolingParams],
410 arrival_time: Optional[float] = None,
411 lora_request: Optional[LoRARequest] = None,
412 trace_headers: Optional[Mapping[str, str]] = None,
413 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
416 - ) -> None:
417 - ...
418 -
419 - @overload
420 - async def add_request_async(
421 - self,
422 - request_id: str,
423 - prompt: PromptType,
424 - params: Union[SamplingParams, PoolingParams],
425 - arrival_time: Optional[float] = None,
426 - lora_request: Optional[LoRARequest] = None,
427 - trace_headers: Optional[Mapping[str, str]] = None,
428 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
429 - ) -> None:
430 - ...
431 -
432 - @deprecate_kwargs(
433 - "inputs",
434 - additional_message="Please use the 'prompt' parameter instead.",
435 - )
436 - async def add_request_async(
437 - self,
438 - request_id: str,
439 - prompt: Optional[PromptType] = None,
440 - params: Optional[Union[SamplingParams, PoolingParams]] = None,
441 - arrival_time: Optional[float] = None,
442 - lora_request: Optional[LoRARequest] = None,
443 - trace_headers: Optional[Mapping[str, str]] = None,
444 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
445 - *,
446 - inputs: Optional[PromptType] = None, # DEPRECATED
414 ) -> None:
415 """Async version of :meth:`add_request`."""
449 - if inputs is not None:
450 - prompt = inputs
451 - assert prompt is not None and params is not None
452 -
416 if lora_request is not None and not self.lora_config:
417 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
418 "not enabled!")
@@ -457,7 +420,7 @@ class _AsyncLLMEngine(LLMEngine):
420 arrival_time = time.time()
421
422 preprocessed_inputs = await self.input_preprocessor.preprocess_async(
460 - prompt,
423 + inputs,
424 request_id=request_id,
425 lora_request=lora_request,
426 prompt_adapter_request=prompt_adapter_request,
@@ -811,55 +774,16 @@ class AsyncLLMEngine:
774
775 # This method does not need to be async, but kept that way
776 # for backwards compatibility.
814 - @overload # DEPRECATED
815 - def add_request(
816 - self,
817 - request_id: str,
818 - *,
819 - inputs: PromptType,
820 - params: Union[SamplingParams, PoolingParams],
821 - arrival_time: Optional[float] = None,
822 - lora_request: Optional[LoRARequest] = None,
823 - trace_headers: Optional[Mapping[str, str]] = None,
824 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
825 - ) -> Coroutine[None, None, AsyncGenerator[Union[
826 - RequestOutput, EmbeddingRequestOutput], None]]:
827 - ...
828 -
829 - @overload
830 - def add_request(
831 - self,
832 - request_id: str,
833 - prompt: PromptType,
834 - params: Union[SamplingParams, PoolingParams],
835 - arrival_time: Optional[float] = None,
836 - lora_request: Optional[LoRARequest] = None,
837 - trace_headers: Optional[Mapping[str, str]] = None,
838 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
839 - ) -> Coroutine[None, None, AsyncGenerator[Union[
840 - RequestOutput, EmbeddingRequestOutput], None]]:
841 - ...
842 -
843 - @deprecate_kwargs(
844 - "inputs",
845 - additional_message="Please use the 'prompt' parameter instead.",
846 - )
777 async def add_request(
778 self,
779 request_id: str,
850 - prompt: Optional[PromptType] = None,
851 - params: Optional[Union[SamplingParams, PoolingParams]] = None,
780 + inputs: PromptInputs,
781 + params: Union[SamplingParams, PoolingParams],
782 arrival_time: Optional[float] = None,
783 lora_request: Optional[LoRARequest] = None,
784 trace_headers: Optional[Mapping[str, str]] = None,
855 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
856 - *,
857 - inputs: Optional[PromptType] = None, # DEPRECATED
785 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
786 ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
859 - if inputs is not None:
860 - prompt = inputs
861 - assert prompt is not None and params is not None
862 -
787 if not self.is_running:
788 if self.start_engine_loop:
789 self.start_background_loop()
@@ -873,7 +797,7 @@ class AsyncLLMEngine:
797 stream = self._request_tracker.add_request(
798 request_id,
799 verbose=self.log_requests,
876 - prompt=prompt,
800 + inputs=inputs,
801 params=params,
802 arrival_time=arrival_time or time.time(),
803 lora_request=lora_request,
@@ -884,7 +808,7 @@ class AsyncLLMEngine:
808
809 async def generate(
810 self,
887 - prompt: PromptType,
811 + inputs: PromptInputs,
812 sampling_params: SamplingParams,
813 request_id: str,
814 lora_request: Optional[LoRARequest] = None,
@@ -898,7 +822,8 @@ class AsyncLLMEngine:
822 from the LLMEngine to the caller.
823
824 Args:
901 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
825 + inputs: The inputs to the LLM. See
826 + :class:`~vllm.inputs.PromptInputs`
827 for more details about the format of each input.
828 sampling_params: The sampling parameters of the request.
829 request_id: The unique id of the request.
@@ -956,7 +881,7 @@ class AsyncLLMEngine:
881 """
882 async for output in await self.add_request(
883 request_id,
959 - prompt,
884 + inputs,
885 sampling_params,
886 lora_request=lora_request,
887 trace_headers=trace_headers,
@@ -966,7 +891,7 @@ class AsyncLLMEngine:
891
892 async def encode(
893 self,
969 - prompt: PromptType,
894 + inputs: PromptInputs,
895 pooling_params: PoolingParams,
896 request_id: str,
897 lora_request: Optional[LoRARequest] = None,
@@ -979,7 +904,8 @@ class AsyncLLMEngine:
904 from the LLMEngine to the caller.
905
906 Args:
982 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
907 + inputs: The inputs to the LLM. See
908 + :class:`~vllm.inputs.PromptInputs`
909 for more details about the format of each input.
910 pooling_params: The pooling parameters of the request.
911 request_id: The unique id of the request.
@@ -1033,7 +959,7 @@ class AsyncLLMEngine:
959 """
960 async for output in await self.add_request(
961 request_id,
1036 - prompt,
962 + inputs,
963 pooling_params,
964 lora_request=lora_request,
965 trace_headers=trace_headers,
vllm/engine/llm_engine.py
+7 -45
@@ -6,7 +6,7 @@ from functools import partial
6 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7 Iterable, List, Mapping, NamedTuple, Optional)
8 from typing import Sequence as GenericSequence
9 -from typing import Set, Type, Union, overload
9 +from typing import Set, Type, Union
10
11 import torch
12 from typing_extensions import TypeVar
@@ -29,7 +29,7 @@ from vllm.executor.executor_base import ExecutorBase
29 from vllm.executor.gpu_executor import GPUExecutor
30 from vllm.executor.ray_utils import initialize_ray_cluster
31 from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32 - InputRegistry, LLMInputs, PromptType)
32 + InputRegistry, LLMInputs, PromptInputs)
33 from vllm.inputs.preprocess import InputPreprocessor
34 from vllm.logger import init_logger
35 from vllm.lora.request import LoRARequest
@@ -51,7 +51,7 @@ from vllm.transformers_utils.tokenizer_group import (
51 BaseTokenizerGroup, init_tokenizer_from_configs)
52 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
53 usage_message)
54 -from vllm.utils import Counter, Device, deprecate_kwargs, weak_bind
54 +from vllm.utils import Counter, Device, weak_bind
55 from vllm.version import __version__ as VLLM_VERSION
56
57 logger = init_logger(__name__)
@@ -689,51 +689,16 @@ class LLMEngine:
689 def stop_remote_worker_execution_loop(self) -> None:
690 self.model_executor.stop_remote_worker_execution_loop()
691
692 - @overload # DEPRECATED
692 def add_request(
693 self,
694 request_id: str,
696 - *,
697 - inputs: PromptType,
695 + inputs: PromptInputs,
696 params: Union[SamplingParams, PoolingParams],
697 arrival_time: Optional[float] = None,
698 lora_request: Optional[LoRARequest] = None,
699 trace_headers: Optional[Mapping[str, str]] = None,
700 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
701 priority: int = 0,
704 - ) -> None:
705 - ...
706 -
707 - @overload
708 - def add_request(
709 - self,
710 - request_id: str,
711 - prompt: PromptType,
712 - params: Union[SamplingParams, PoolingParams],
713 - arrival_time: Optional[float] = None,
714 - lora_request: Optional[LoRARequest] = None,
715 - trace_headers: Optional[Mapping[str, str]] = None,
716 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
717 - priority: int = 0,
718 - ) -> None:
719 - ...
720 -
721 - @deprecate_kwargs(
722 - "inputs",
723 - additional_message="Please use the 'prompt' parameter instead.",
724 - )
725 - def add_request(
726 - self,
727 - request_id: str,
728 - prompt: Optional[PromptType] = None,
729 - params: Optional[Union[SamplingParams, PoolingParams]] = None,
730 - arrival_time: Optional[float] = None,
731 - lora_request: Optional[LoRARequest] = None,
732 - trace_headers: Optional[Mapping[str, str]] = None,
733 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
734 - priority: int = 0,
735 - *,
736 - inputs: Optional[PromptType] = None, # DEPRECATED
702 ) -> None:
703 """Add a request to the engine's request pool.
704
@@ -743,7 +708,8 @@ class LLMEngine:
708
709 Args:
710 request_id: The unique ID of the request.
746 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
711 + inputs: The inputs to the LLM. See
712 + :class:`~vllm.inputs.PromptInputs`
713 for more details about the format of each input.
714 params: Parameters for sampling or pooling.
715 :class:`~vllm.SamplingParams` for text generation.
@@ -778,10 +744,6 @@ class LLMEngine:
744 >>> # continue the request processing
745 >>> ...
746 """
781 - if inputs is not None:
782 - prompt = inputs
783 - assert prompt is not None and params is not None
784 -
747 if lora_request is not None and not self.lora_config:
748 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
749 "not enabled!")
@@ -794,7 +756,7 @@ class LLMEngine:
756 arrival_time = time.time()
757
758 preprocessed_inputs = self.input_preprocessor.preprocess(
797 - prompt,
759 + inputs,
760 request_id=request_id,
761 lora_request=lora_request,
762 prompt_adapter_request=prompt_adapter_request,
vllm/engine/multiprocessing/__init__.py
+3 -58
@@ -1,14 +1,13 @@
1 from dataclasses import dataclass
2 from enum import Enum
3 -from typing import List, Mapping, Optional, Union, overload
3 +from typing import List, Mapping, Optional, Union
4
5 from vllm import PoolingParams
6 -from vllm.inputs import PromptType
6 +from vllm.inputs import PromptInputs
7 from vllm.lora.request import LoRARequest
8 from vllm.outputs import RequestOutput
9 from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.sampling_params import SamplingParams
11 -from vllm.utils import deprecate_kwargs
11
12 VLLM_RPC_SUCCESS_STR = "SUCCESS"
13
@@ -24,67 +23,13 @@ class MQEngineDeadError(RuntimeError):
23
24 @dataclass
25 class RPCProcessRequest:
27 - prompt: PromptType
26 + inputs: PromptInputs
27 params: Union[SamplingParams, PoolingParams]
28 request_id: str
29 lora_request: Optional[LoRARequest] = None
30 trace_headers: Optional[Mapping[str, str]] = None
31 prompt_adapter_request: Optional[PromptAdapterRequest] = None
32
34 - @overload # DEPRECATED
35 - def __init__(
36 - self,
37 - *,
38 - inputs: PromptType,
39 - params: Union[SamplingParams, PoolingParams],
40 - request_id: str,
41 - lora_request: Optional[LoRARequest] = None,
42 - trace_headers: Optional[Mapping[str, str]] = None,
43 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
44 - ) -> None:
45 - ...
46 -
47 - @overload
48 - def __init__(
49 - self,
50 - prompt: PromptType,
51 - params: Union[SamplingParams, PoolingParams],
52 - request_id: str,
53 - lora_request: Optional[LoRARequest] = None,
54 - trace_headers: Optional[Mapping[str, str]] = None,
55 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
56 - ) -> None:
57 - ...
58 -
59 - @deprecate_kwargs(
60 - "inputs",
61 - additional_message="Please use the 'prompt' parameter instead.",
62 - )
63 - def __init__(
64 - self,
65 - prompt: Optional[PromptType] = None,
66 - params: Optional[Union[SamplingParams, PoolingParams]] = None,
67 - request_id: Optional[str] = None,
68 - lora_request: Optional[LoRARequest] = None,
69 - trace_headers: Optional[Mapping[str, str]] = None,
70 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
71 - *,
72 - inputs: Optional[PromptType] = None, # DEPRECATED
73 - ) -> None:
74 - if inputs is not None:
75 - prompt = inputs
76 - assert (prompt is not None and params is not None
77 - and request_id is not None)
78 -
79 - super().__init__()
80 -
81 - self.prompt = prompt
82 - self.params = params
83 - self.request_id = request_id
84 - self.lora_request = lora_request
85 - self.trace_headers = trace_headers
86 - self.prompt_adapter_request = prompt_adapter_request
87 -
33
34 @dataclass
35 class RPCError:
vllm/engine/multiprocessing/client.py
+13 -82
@@ -3,7 +3,7 @@ import copy
3 import pickle
4 from contextlib import contextmanager, suppress
5 from typing import (Any, AsyncGenerator, Dict, Iterator, Mapping, Optional,
6 - Union, overload)
6 + Union)
7
8 import cloudpickle
9 import zmq
@@ -25,14 +25,13 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25 RPCUProfileRequest)
26 # yapf: enable
27 from vllm.envs import VLLM_RPC_TIMEOUT
28 -from vllm.inputs import PromptType
28 +from vllm.inputs import PromptInputs
29 from vllm.logger import init_logger
30 from vllm.lora.request import LoRARequest
31 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
32 from vllm.prompt_adapter.request import PromptAdapterRequest
33 from vllm.sampling_params import SamplingParams
34 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
35 -from vllm.utils import deprecate_kwargs
35
36 logger = init_logger(__name__)
37
@@ -368,45 +367,14 @@ class MQLLMEngineClient:
367 def dead_error(self) -> BaseException:
368 return ENGINE_DEAD_ERROR(self._errored_with)
369
371 - @overload # DEPRECATED
370 def generate(
371 self,
374 - *,
375 - inputs: PromptType,
372 + inputs: PromptInputs,
373 sampling_params: SamplingParams,
374 request_id: str,
375 lora_request: Optional[LoRARequest] = None,
376 trace_headers: Optional[Mapping[str, str]] = None,
380 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
381 - ) -> AsyncGenerator[RequestOutput, None]:
382 - ...
383 -
384 - @overload
385 - def generate(
386 - self,
387 - prompt: PromptType,
388 - sampling_params: SamplingParams,
389 - request_id: str,
390 - lora_request: Optional[LoRARequest] = None,
391 - trace_headers: Optional[Mapping[str, str]] = None,
392 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
393 - ) -> AsyncGenerator[RequestOutput, None]:
394 - ...
395 -
396 - @deprecate_kwargs(
397 - "inputs",
398 - additional_message="Please use the 'prompt' parameter instead.",
399 - )
400 - def generate(
401 - self,
402 - prompt: Optional[PromptType] = None,
403 - sampling_params: Optional[SamplingParams] = None,
404 - request_id: Optional[str] = None,
405 - lora_request: Optional[LoRARequest] = None,
406 - trace_headers: Optional[Mapping[str, str]] = None,
407 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
408 - *,
409 - inputs: Optional[PromptType] = None # DEPRECATED
377 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
378 ) -> AsyncGenerator[RequestOutput, None]:
379 """Generate outputs for a request.
380
@@ -415,7 +383,8 @@ class MQLLMEngineClient:
383 from the LLMEngine to the caller.
384
385 Args:
418 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
386 + inputs: The inputs to the LLM. See
387 + :class:`~vllm.inputs.PromptInputs`
388 for more details about the format of each input.
389 sampling_params: The sampling parameters of the request.
390 request_id: The unique id of the request.
@@ -424,51 +393,17 @@ class MQLLMEngineClient:
393 prompt_adapter_request: Prompt Adapter request to use
394 for generation, if any.
395 """
427 - if inputs is not None:
428 - prompt = inputs
429 - assert (prompt is not None and sampling_params is not None
430 - and request_id is not None)
431 -
432 - return self._process_request(prompt, sampling_params, request_id,
396 + return self._process_request(inputs, sampling_params, request_id,
397 lora_request, trace_headers,
398 prompt_adapter_request)
399
436 - @overload # DEPRECATED
400 def encode(
401 self,
439 - *,
440 - inputs: PromptType,
402 + inputs: PromptInputs,
403 pooling_params: PoolingParams,
404 request_id: str,
405 lora_request: Optional[LoRARequest] = None,
406 trace_headers: Optional[Mapping[str, str]] = None,
445 - ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
446 - ...
447 -
448 - @overload
449 - def encode(
450 - self,
451 - prompt: PromptType,
452 - pooling_params: PoolingParams,
453 - request_id: str,
454 - lora_request: Optional[LoRARequest] = None,
455 - trace_headers: Optional[Mapping[str, str]] = None,
456 - ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
457 - ...
458 -
459 - @deprecate_kwargs(
460 - "inputs",
461 - additional_message="Please use the 'prompt' parameter instead.",
462 - )
463 - def encode(
464 - self,
465 - prompt: Optional[PromptType] = None,
466 - pooling_params: Optional[PoolingParams] = None,
467 - request_id: Optional[str] = None,
468 - lora_request: Optional[LoRARequest] = None,
469 - trace_headers: Optional[Mapping[str, str]] = None,
470 - *,
471 - inputs: Optional[PromptType] = None # DEPRECATED
407 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
408 """Generate outputs for a request from an embedding model.
409
@@ -477,7 +412,8 @@ class MQLLMEngineClient:
412 from the LLMEngine to the caller.
413
414 Args:
480 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
415 + inputs: The inputs to the LLM. See
416 + :class:`~vllm.inputs.PromptInputs`
417 for more details about the format of each input.
418 pooling_params: The pooling parameters of the request.
419 request_id: The unique id of the request.
@@ -488,17 +424,12 @@ class MQLLMEngineClient:
424 The output `EmbeddingRequestOutput` objects from the LLMEngine
425 for the request.
426 """
491 - if inputs is not None:
492 - prompt = inputs
493 - assert (prompt is not None and pooling_params is not None
494 - and request_id is not None)
495 -
496 - return self._process_request(prompt, pooling_params, request_id,
427 + return self._process_request(inputs, pooling_params, request_id,
428 lora_request, trace_headers)
429
430 async def _process_request(
431 self,
501 - prompt: PromptType,
432 + inputs: PromptInputs,
433 params: Union[SamplingParams, PoolingParams],
434 request_id: str,
435 lora_request: Optional[LoRARequest] = None,
@@ -531,7 +462,7 @@ class MQLLMEngineClient:
462
463 request_bytes = pickle.dumps(
464 RPCProcessRequest(
534 - prompt=prompt,
465 + inputs=inputs,
466 params=params,
467 request_id=request_id,
468 lora_request=lora_request,
vllm/engine/multiprocessing/engine.py
+1 -1
@@ -278,7 +278,7 @@ class MQLLMEngine:
278 try:
279 self.engine.add_request(
280 request_id=request_id,
281 - prompt=request.prompt,
281 + inputs=request.inputs,
282 params=request.params,
283 lora_request=request.lora_request,
284 trace_headers=request.trace_headers,
vllm/engine/protocol.py
+4 -4
@@ -3,7 +3,7 @@ from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
3
4 from vllm.config import DecodingConfig, ModelConfig
5 from vllm.core.scheduler import SchedulerOutputs
6 -from vllm.inputs.data import PromptType
6 +from vllm.inputs.data import PromptInputs
7 from vllm.lora.request import LoRARequest
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -35,19 +35,19 @@ class EngineClient(Protocol):
35
36 def generate(
37 self,
38 - prompt: PromptType,
38 + inputs: PromptInputs,
39 sampling_params: SamplingParams,
40 request_id: str,
41 lora_request: Optional[LoRARequest] = None,
42 trace_headers: Optional[Mapping[str, str]] = None,
43 prompt_adapter_request: Optional[PromptAdapterRequest] = None
44 ) -> AsyncGenerator[RequestOutput, None]:
45 - """Generate outputs for a request."""
45 + """Generates outputs for a request"""
46 ...
47
48 def encode(
49 self,
50 - prompt: PromptType,
50 + inputs: PromptInputs,
51 pooling_params: PoolingParams,
52 request_id: str,
53 lora_request: Optional[LoRARequest] = None,
vllm/entrypoints/llm.py
+33 -35
@@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
12 apply_hf_chat_template,
13 apply_mistral_chat_template,
14 parse_chat_messages)
15 -from vllm.inputs import PromptType, TextPrompt, TokensPrompt
15 +from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
16 from vllm.inputs.parse import parse_and_batch_prompt
17 from vllm.logger import init_logger
18 from vllm.lora.request import LoRARequest
@@ -293,8 +293,8 @@ class LLM:
293 @overload
294 def generate(
295 self,
296 - prompts: Union[PromptType, Sequence[PromptType]],
297 - /,
296 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
297 + /, # We may enable `inputs` keyword after removing the old API
298 *,
299 sampling_params: Optional[Union[SamplingParams,
300 Sequence[SamplingParams]]] = None,
@@ -304,13 +304,14 @@ class LLM:
304 ...
305
306 @deprecate_kwargs(
307 + "prompts",
308 "prompt_token_ids",
309 is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
309 - additional_message="Please use the 'prompts' parameter instead.",
310 + additional_message="Please use the 'inputs' parameter instead.",
311 )
312 def generate(
313 self,
313 - prompts: Union[Union[PromptType, Sequence[PromptType]],
314 + prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
315 Optional[Union[str, List[str]]]] = None,
316 sampling_params: Optional[Union[SamplingParams,
317 Sequence[SamplingParams]]] = None,
@@ -329,9 +330,7 @@ class LLM:
330 into a single list and pass it to this method.
331
332 Args:
332 - prompts: The prompts to the LLM. You may pass a sequence of prompts
333 - for batch inference. See :class:`~vllm.inputs.PromptType`
334 - for more details about the format of each prompts.
333 + inputs: A list of inputs to generate completions for.
334 sampling_params: The sampling parameters for text generation. If
335 None, we use the default sampling parameters.
336 When it is a single value, it is applied to every prompt.
@@ -359,13 +358,12 @@ class LLM:
358 "models (XForCausalLM, XForConditionalGeneration).")
359
360 if prompt_token_ids is not None:
362 - parsed_prompts = self._convert_v1_inputs(
361 + inputs = self._convert_v1_inputs(
362 prompts=cast(Optional[Union[str, List[str]]], prompts),
363 prompt_token_ids=prompt_token_ids,
364 )
365 else:
367 - parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
368 - prompts)
366 + inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
367
368 if isinstance(guided_options_request, dict):
369 if len(guided_options_request) > 1:
@@ -380,7 +378,7 @@ class LLM:
378 sampling_params = SamplingParams()
379
380 self._validate_and_add_requests(
383 - prompts=parsed_prompts,
381 + inputs=inputs,
382 params=sampling_params,
383 lora_request=lora_request,
384 prompt_adapter_request=prompt_adapter_request,
@@ -650,8 +648,8 @@ class LLM:
648 @overload
649 def encode(
650 self,
653 - prompts: Union[PromptType, Sequence[PromptType]],
654 - /,
651 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
652 + /, # We may enable `inputs` keyword after removing the old API
653 *,
654 pooling_params: Optional[Union[PoolingParams,
655 Sequence[PoolingParams]]] = None,
@@ -661,13 +659,14 @@ class LLM:
659 ...
660
661 @deprecate_kwargs(
662 + "prompts",
663 "prompt_token_ids",
664 is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
666 - additional_message="Please use the 'prompts' parameter instead.",
665 + additional_message="Please use the 'inputs' parameter instead.",
666 )
667 def encode(
668 self,
670 - prompts: Union[Union[PromptType, Sequence[PromptType]],
669 + prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
670 Optional[Union[str, List[str]]]] = None,
671 pooling_params: Optional[Union[PoolingParams,
672 Sequence[PoolingParams]]] = None,
@@ -683,9 +682,9 @@ class LLM:
682 into a single list and pass it to this method.
683
684 Args:
686 - prompts: The prompts to the LLM. You may pass a sequence of prompts
687 - for batch inference. See :class:`~vllm.inputs.PromptType`
688 - for more details about the format of each prompts.
685 + inputs: The inputs to the LLM. You may pass a sequence of inputs for
686 + batch inference. See :class:`~vllm.inputs.PromptInputs`
687 + for more details about the format of each input.
688 pooling_params: The pooling parameters for pooling. If None, we
689 use the default pooling parameters.
690 use_tqdm: Whether to use tqdm to display the progress bar.
@@ -708,20 +707,19 @@ class LLM:
707 )
708
709 if prompt_token_ids is not None:
711 - parsed_prompts = self._convert_v1_inputs(
710 + inputs = self._convert_v1_inputs(
711 prompts=cast(Optional[Union[str, List[str]]], prompts),
712 prompt_token_ids=prompt_token_ids,
713 )
714 else:
716 - parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
717 - prompts)
715 + inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
716
717 if pooling_params is None:
718 # Use default pooling params.
719 pooling_params = PoolingParams()
720
721 self._validate_and_add_requests(
724 - prompts=parsed_prompts,
722 + inputs=inputs,
723 params=pooling_params,
724 lora_request=lora_request,
725 prompt_adapter_request=prompt_adapter_request,
@@ -765,9 +763,9 @@ class LLM:
763 raise ValueError("Either prompts or prompt_token_ids must be "
764 "provided.")
765
768 - parsed_prompts: List[PromptType] = []
766 + inputs: List[PromptInputs] = []
767 for i in range(num_requests):
770 - item: PromptType
768 + item: PromptInputs
769
770 if prompts is not None:
771 item = TextPrompt(prompt=prompts[i])
@@ -776,13 +774,13 @@ class LLM:
774 else:
775 raise AssertionError
776
779 - parsed_prompts.append(item)
777 + inputs.append(item)
778
781 - return parsed_prompts
779 + return inputs
780
781 def _validate_and_add_requests(
782 self,
785 - prompts: Union[PromptType, Sequence[PromptType]],
783 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
784 params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
785 Sequence[PoolingParams]],
786 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
@@ -790,11 +788,11 @@ class LLM:
788 guided_options: Optional[GuidedDecodingRequest] = None,
789 priority: Optional[List[int]] = None,
790 ) -> None:
793 - if isinstance(prompts, (str, dict)):
791 + if isinstance(inputs, (str, dict)):
792 # Convert a single prompt to a list.
795 - prompts = [prompts]
793 + inputs = [inputs]
794
797 - num_requests = len(prompts)
795 + num_requests = len(inputs)
796 if isinstance(params, list) and len(params) != num_requests:
797 raise ValueError("The lengths of prompts and params "
798 "must be the same.")
@@ -811,9 +809,9 @@ class LLM:
809 sp.output_kind = RequestOutputKind.FINAL_ONLY
810
811 # Add requests to the engine.
814 - for i, prompt in enumerate(prompts):
812 + for i, request_inputs in enumerate(inputs):
813 self._add_request(
816 - prompt,
814 + request_inputs,
815 params[i] if isinstance(params, Sequence) else params,
816 lora_request=lora_request[i] if isinstance(
817 lora_request, Sequence) else lora_request,
@@ -823,7 +821,7 @@ class LLM:
821
822 def _add_request(
823 self,
826 - prompt: PromptType,
824 + inputs: PromptInputs,
825 params: Union[SamplingParams, PoolingParams],
826 lora_request: Optional[LoRARequest] = None,
827 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -832,7 +830,7 @@ class LLM:
830 request_id = str(next(self.request_counter))
831 self.llm_engine.add_request(
832 request_id,
835 - prompt,
833 + inputs,
834 params,
835 lora_request=lora_request,
836 prompt_adapter_request=prompt_adapter_request,
vllm/inputs/__init__.py
+3 -17
@@ -1,5 +1,5 @@
1 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2 - LLMInputs, PromptType, SingletonPrompt, TextPrompt,
2 + LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
3 TokensPrompt, build_explicit_enc_dec_prompt,
4 to_enc_dec_tuple_list, zip_enc_dec_prompts)
5 from .registry import InputContext, InputRegistry
@@ -16,8 +16,8 @@ See also:
16 __all__ = [
17 "TextPrompt",
18 "TokensPrompt",
19 - "PromptType",
20 - "SingletonPrompt",
19 + "PromptInputs",
20 + "SingletonPromptInputs",
21 "ExplicitEncoderDecoderPrompt",
22 "LLMInputs",
23 "EncoderDecoderLLMInputs",
@@ -28,17 +28,3 @@ __all__ = [
28 "InputContext",
29 "InputRegistry",
30 ]
31 -
32 -
33 -def __getattr__(name: str):
34 - if name == "PromptInput":
35 - import warnings
36 -
37 - msg = ("PromptInput has been renamed to PromptType. "
38 - "The original name will be removed in an upcoming version.")
39 -
40 - warnings.warn(DeprecationWarning(msg), stacklevel=2)
41 -
42 - return PromptType
43 -
44 - raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/data.py
+19 -29
@@ -33,7 +33,7 @@ class TokensPrompt(TypedDict):
33 """
34
35
36 -SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
36 +SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
37 """
38 Set of possible schemas for a single LLM input:
39
@@ -46,7 +46,7 @@ which may be utilized for encoder/decoder models when
46 the user desires to express both the encoder & decoder
47 prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
49 -A prompt of type :class:`SingletonPrompt` may be employed
49 +A prompt of type :class:`SingletonPromptInputs` may be employed
50 as (1) input to a decoder-only model, (2) input to
51 the encoder of an encoder/decoder model, in the scenario
52 where the decoder-prompt is not specified explicitly, or
@@ -55,33 +55,33 @@ more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55 """
56
57 _T1_co = TypeVar("_T1_co",
58 - bound=SingletonPrompt,
59 - default=SingletonPrompt,
58 + bound=SingletonPromptInputs,
59 + default=SingletonPromptInputs,
60 covariant=True)
61 _T2_co = TypeVar("_T2_co",
62 - bound=SingletonPrompt,
63 - default=SingletonPrompt,
62 + bound=SingletonPromptInputs,
63 + default=SingletonPromptInputs,
64 covariant=True)
65
66
67 # TODO: Make fields ReadOnly once mypy supports it
68 class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
69 - """
70 - Represents an encoder/decoder model input prompt,
71 - comprising an explicit encoder prompt and a decoder prompt.
69 + """Represents an encoder/decoder model input prompt,
70 + comprising an explicit encoder prompt and a
71 + decoder prompt.
72
73 The encoder and decoder prompts, respectively,
74 may formatted according to any of the
75 - :class:`SingletonPrompt` schemas, and are not
75 + :class:`SingletonPromptInputs` schemas, and are not
76 required to have the same schema.
77
78 Only the encoder prompt may have multi-modal data.
79
80 Note that an :class:`ExplicitEncoderDecoderPrompt` may not
81 be used as an input to a decoder-only model,
82 - and that the :code:`encoder_prompt` and :code:`decoder_prompt`
82 + and that the `encoder_prompt` and `decoder_prompt`
83 fields of this data structure themselves must be
84 - :class:`SingletonPrompt` instances.
84 + :class:`SingletonPromptInputs` instances.
85 """
86
87 encoder_prompt: _T1_co
@@ -89,7 +89,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
89 decoder_prompt: Optional[_T2_co]
90
91
92 -PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
92 +PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
93 """
94 Set of possible schemas for an LLM input, including
95 both decoder-only and encoder/decoder input types:
@@ -140,8 +140,12 @@ class EncoderDecoderLLMInputs(LLMInputs):
140 """
141
142
143 -_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
144 -_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
143 +_T1 = TypeVar("_T1",
144 + bound=SingletonPromptInputs,
145 + default=SingletonPromptInputs)
146 +_T2 = TypeVar("_T2",
147 + bound=SingletonPromptInputs,
148 + default=SingletonPromptInputs)
149
150
151 def build_explicit_enc_dec_prompt(
@@ -172,17 +176,3 @@ def to_enc_dec_tuple_list(
176 return [(enc_dec_prompt["encoder_prompt"],
177 enc_dec_prompt["decoder_prompt"])
178 for enc_dec_prompt in enc_dec_prompts]
175 -
176 -
177 -def __getattr__(name: str):
178 - if name == "PromptInput":
179 - import warnings
180 -
181 - msg = ("PromptInput has been renamed to PromptType. "
182 - "The original name will be removed in an upcoming version.")
183 -
184 - warnings.warn(DeprecationWarning(msg), stacklevel=2)
185 -
186 - return PromptType
187 -
188 - raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/inputs/parse.py
+11 -11
@@ -5,7 +5,7 @@ from typing_extensions import TypeIs
5 from vllm.utils import is_list_of
6
7 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8 - LLMInputs, PromptType, SingletonPrompt, TextPrompt,
8 + LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
9 TokensPrompt)
10
11
@@ -81,23 +81,23 @@ class ParsedTokensPrompt(TypedDict):
81
82
83 def parse_singleton_prompt(
84 - prompt: SingletonPrompt,
84 + inputs: SingletonPromptInputs,
85 ) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86 - if isinstance(prompt, str):
87 - return ParsedStrPrompt(type="str", content=prompt)
88 - elif isinstance(prompt, dict):
89 - if "prompt_token_ids" in prompt:
86 + if isinstance(inputs, str):
87 + return ParsedStrPrompt(type="str", content=inputs)
88 + elif isinstance(inputs, dict):
89 + if "prompt_token_ids" in inputs:
90 return ParsedTokensPrompt(type="tokens",
91 - content=prompt) # type: ignore
92 - elif "prompt" in prompt:
93 - return ParsedTextPrompt(type="text", content=prompt)
91 + content=inputs) # type: ignore
92 + elif "prompt" in inputs:
93 + return ParsedTextPrompt(type="text", content=inputs)
94
95 raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98 def is_explicit_encoder_decoder_prompt(
99 - prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 - return isinstance(prompt, dict) and "encoder_prompt" in prompt
99 + inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 + return isinstance(inputs, dict) and "encoder_prompt" in inputs
101
102
103 def is_valid_encoder_decoder_llm_inputs(
vllm/inputs/preprocess.py
+43 -43
@@ -9,8 +9,8 @@ from vllm.lora.request import LoRARequest
9 from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
12 -from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
13 - SingletonPrompt)
12 +from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
13 + SingletonPromptInputs)
14 from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
15
16 if TYPE_CHECKING:
@@ -206,7 +206,7 @@ class InputPreprocessor:
206
207 def _extract_prompt_components(
208 self,
209 - prompt: SingletonPrompt,
209 + inputs: SingletonPromptInputs,
210 request_id: str,
211 lora_request: Optional[LoRARequest] = None,
212 ) -> PromptComponents:
@@ -216,7 +216,7 @@ class InputPreprocessor:
216 Arguments:
217
218 * request_id
219 - * prompt: single encoder or decoder input prompt
219 + * inputs: single encoder or decoder input prompt
220 * lora_request: this is only valid for decoder prompts
221
222 Returns:
@@ -226,24 +226,24 @@ class InputPreprocessor:
226 * multi_modal_data
227 '''
228
229 - parsed = parse_singleton_prompt(prompt)
229 + parsed = parse_singleton_prompt(inputs)
230
231 if parsed["type"] == "str":
232 - prompt_text = parsed["content"]
232 + prompt = parsed["content"]
233 prompt_token_ids = self._tokenize_prompt(
234 - prompt_text,
234 + prompt,
235 request_id=request_id,
236 lora_request=lora_request,
237 )
238 multi_modal_data = None
239 elif parsed["type"] == "tokens":
240 - prompt_text = None
240 + prompt = None
241 prompt_token_ids = parsed["content"]["prompt_token_ids"]
242 multi_modal_data = parsed["content"].get("multi_modal_data")
243 elif parsed["type"] == "text":
244 - prompt_text = parsed["content"]["prompt"]
244 + prompt = parsed["content"]["prompt"]
245 prompt_token_ids = self._tokenize_prompt(
246 - prompt_text,
246 + prompt,
247 request_id=request_id,
248 lora_request=lora_request,
249 )
@@ -251,33 +251,33 @@ class InputPreprocessor:
251 else:
252 assert_never(parsed)
253
254 - return prompt_text, prompt_token_ids, multi_modal_data
254 + return prompt, prompt_token_ids, multi_modal_data
255
256 async def _extract_prompt_components_async(
257 self,
258 - prompt: SingletonPrompt,
258 + inputs: SingletonPromptInputs,
259 request_id: str,
260 lora_request: Optional[LoRARequest] = None,
261 ) -> PromptComponents:
262 """Async version of :meth:`_extract_prompt_components`."""
263 - parsed = parse_singleton_prompt(prompt)
263 + parsed = parse_singleton_prompt(inputs)
264
265 if parsed["type"] == "str":
266 - prompt_text = parsed["content"]
266 + prompt = parsed["content"]
267 prompt_token_ids = await self._tokenize_prompt_async(
268 - prompt_text,
268 + prompt,
269 request_id=request_id,
270 lora_request=lora_request,
271 )
272 multi_modal_data = None
273 elif parsed["type"] == "tokens":
274 - prompt_text = None
274 + prompt = None
275 prompt_token_ids = parsed["content"]["prompt_token_ids"]
276 multi_modal_data = parsed["content"].get("multi_modal_data")
277 elif parsed["type"] == "text":
278 - prompt_text = parsed["content"]["prompt"]
278 + prompt = parsed["content"]["prompt"]
279 prompt_token_ids = await self._tokenize_prompt_async(
280 - prompt_text,
280 + prompt,
281 request_id=request_id,
282 lora_request=lora_request,
283 )
@@ -285,7 +285,7 @@ class InputPreprocessor:
285 else:
286 assert_never(parsed)
287
288 - return prompt_text, prompt_token_ids, multi_modal_data
288 + return prompt, prompt_token_ids, multi_modal_data
289
290 def _build_enc_dec_llm_inputs(
291 self,
@@ -311,7 +311,7 @@ class InputPreprocessor:
311
312 def _process_encoder_decoder_prompt(
313 self,
314 - prompt: PromptType,
314 + inputs: PromptInputs,
315 request_id: str,
316 ) -> EncoderDecoderLLMInputs:
317 '''
@@ -339,7 +339,7 @@ class InputPreprocessor:
339
340 Arguments:
341
342 - * prompt: an input prompt
342 + * inputs: an input prompt
343 * request_id
344
345 Returns:
@@ -350,13 +350,13 @@ class InputPreprocessor:
350 encoder_comps: PromptComponents
351 decoder_comps: DecoderPromptComponents
352
353 - if is_explicit_encoder_decoder_prompt(prompt):
353 + if is_explicit_encoder_decoder_prompt(inputs):
354 encoder_comps = self._extract_prompt_components(
355 - prompt["encoder_prompt"],
355 + inputs["encoder_prompt"],
356 request_id=request_id,
357 )
358
359 - if (decoder_input := prompt["decoder_prompt"]) is None:
359 + if (decoder_input := inputs["decoder_prompt"]) is None:
360 decoder_comps = None, None, None
361 else:
362 decoder_comps = self._extract_prompt_components(
@@ -365,7 +365,7 @@ class InputPreprocessor:
365 )
366 else:
367 encoder_comps = self._extract_prompt_components(
368 - prompt,
368 + inputs,
369 request_id=request_id,
370 )
371
@@ -375,20 +375,20 @@ class InputPreprocessor:
375
376 async def _process_encoder_decoder_prompt_async(
377 self,
378 - prompt: PromptType,
378 + inputs: PromptInputs,
379 request_id: str,
380 ) -> EncoderDecoderLLMInputs:
381 """Async version of :meth:`_process_encoder_decoder_prompt`."""
382 encoder_comps: PromptComponents
383 decoder_comps: DecoderPromptComponents
384
385 - if is_explicit_encoder_decoder_prompt(prompt):
385 + if is_explicit_encoder_decoder_prompt(inputs):
386 encoder_task = self._extract_prompt_components_async(
387 - prompt["encoder_prompt"],
387 + inputs["encoder_prompt"],
388 request_id=request_id,
389 )
390
391 - if (decoder_input := prompt["decoder_prompt"]) is None:
391 + if (decoder_input := inputs["decoder_prompt"]) is None:
392 encoder_comps = await encoder_task
393 decoder_comps = None, None, None
394 else:
@@ -401,7 +401,7 @@ class InputPreprocessor:
401 encoder_task, decoder_task)
402 else:
403 encoder_comps = await self._extract_prompt_components_async(
404 - prompt,
404 + inputs,
405 request_id=request_id,
406 )
407
@@ -425,7 +425,7 @@ class InputPreprocessor:
425
426 def _process_decoder_only_prompt(
427 self,
428 - prompt: SingletonPrompt,
428 + inputs: SingletonPromptInputs,
429 request_id: str,
430 lora_request: Optional[LoRARequest] = None,
431 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -436,7 +436,7 @@ class InputPreprocessor:
436
437 Arguments:
438
439 - * prompt: input prompt
439 + * inputs: input prompt
440 * request_id
441 * lora_request
442 * prompt_adapter_request
@@ -447,7 +447,7 @@ class InputPreprocessor:
447 '''
448
449 prompt_comps = self._extract_prompt_components(
450 - prompt,
450 + inputs,
451 request_id=request_id,
452 lora_request=lora_request,
453 )
@@ -459,14 +459,14 @@ class InputPreprocessor:
459
460 async def _process_decoder_only_prompt_async(
461 self,
462 - prompt: SingletonPrompt,
462 + inputs: SingletonPromptInputs,
463 request_id: str,
464 lora_request: Optional[LoRARequest] = None,
465 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
466 ) -> LLMInputs:
467 """Async version of :meth:`_process_decoder_only_prompt`."""
468 prompt_comps = await self._extract_prompt_components_async(
469 - prompt,
469 + inputs,
470 request_id=request_id,
471 lora_request=lora_request,
472 )
@@ -478,7 +478,7 @@ class InputPreprocessor:
478
479 def preprocess(
480 self,
481 - prompt: PromptType,
481 + inputs: PromptInputs,
482 request_id: str,
483 lora_request: Optional[LoRARequest] = None,
484 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -488,17 +488,17 @@ class InputPreprocessor:
488 # Encoder-decoder model requires special mapping of
489 # input prompts to encoder & decoder
490 return self._process_encoder_decoder_prompt(
491 - prompt,
491 + inputs,
492 request_id=request_id,
493 )
494
495 - if is_explicit_encoder_decoder_prompt(prompt):
495 + if is_explicit_encoder_decoder_prompt(inputs):
496 raise ValueError("Cannot pass encoder-decoder prompt "
497 "to decoder-only models")
498
499 # Decoder-only operation
500 return self._process_decoder_only_prompt(
501 - prompt,
501 + inputs,
502 request_id=request_id,
503 lora_request=lora_request,
504 prompt_adapter_request=prompt_adapter_request,
@@ -506,7 +506,7 @@ class InputPreprocessor:
506
507 async def preprocess_async(
508 self,
509 - prompt: PromptType,
509 + inputs: PromptInputs,
510 request_id: str,
511 lora_request: Optional[LoRARequest] = None,
512 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -516,17 +516,17 @@ class InputPreprocessor:
516 # Encoder-decoder model requires special mapping of
517 # input prompts to encoder & decoder
518 return await self._process_encoder_decoder_prompt_async(
519 - prompt,
519 + inputs,
520 request_id=request_id,
521 )
522
523 - if is_explicit_encoder_decoder_prompt(prompt):
523 + if is_explicit_encoder_decoder_prompt(inputs):
524 raise ValueError("Cannot pass encoder-decoder prompt "
525 "to decoder-only models")
526
527 # Decoder-only operation
528 return await self._process_decoder_only_prompt_async(
529 - prompt,
529 + inputs,
530 request_id=request_id,
531 lora_request=lora_request,
532 prompt_adapter_request=prompt_adapter_request,