@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3185fb0cca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

Revert "[Core] Rename `PromptInputs` to `PromptType`, and `inputs` to `prompt`" (#8750)

Simon Mo committed Sep 23, 2024 at 22:45 UTC 3185fb0ccae73816018d0936c03171b7cf1ba2f8
18 files changed +162 -157
benchmarks/benchmark_latency.py
+4 -4
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
13 from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14 -from vllm.inputs import PromptType
14 +from vllm.inputs import PromptInputs
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61 dummy_prompt_token_ids = np.random.randint(10000,
62 size=(args.batch_size,
63 args.input_len))
64 - dummy_prompts: List[PromptType] = [{
64 + dummy_inputs: List[PromptInputs] = [{
65 "prompt_token_ids": batch
66 } for batch in dummy_prompt_token_ids.tolist()]
67
@@ -74,13 +74,13 @@ def main(args: argparse.Namespace):
74 ],
75 on_trace_ready=torch.profiler.tensorboard_trace_handler(
76 str(profile_dir))) as p:
77 - llm.generate(dummy_prompts,
77 + llm.generate(dummy_inputs,
78 sampling_params=sampling_params,
79 use_tqdm=False)
80 print(p.key_averages())
81 else:
82 start_time = time.perf_counter()
83 - llm.generate(dummy_prompts,
83 + llm.generate(dummy_inputs,
84 sampling_params=sampling_params,
85 use_tqdm=False)
86 end_time = time.perf_counter()
docs/source/dev/multimodal/multimodal_index.rst
+1 -1
@@ -8,7 +8,7 @@ Multi-Modality
8 vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10 Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11 -via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
11 +via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
12
13 Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14 by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1 -1
@@ -1,7 +1,7 @@
1 LLM Inputs
2 ==========
3
4 -.. autodata:: vllm.inputs.PromptType
4 +.. autodata:: vllm.inputs.PromptInputs
5
6 .. autoclass:: vllm.inputs.TextPrompt
7 :show-inheritance:
docs/source/models/vlm.rst
+1 -1
@@ -27,7 +27,7 @@ The :class:`~vllm.LLM` class can be instantiated in much the same way as languag
27 We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28 the above snippet. Specifically, ``image_feature_size`` can no longer be specified as we now calculate that internally for each model.
29
30 -To pass an image to the model, note the following in :class:`vllm.inputs.PromptType`:
30 +To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
31
32 * ``prompt``: The prompt should follow the format that is documented on HuggingFace.
33 * ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/mq_llm_engine/test_error_handling.py
+6 -6
@@ -61,7 +61,7 @@ async def test_evil_forward(tmp_socket):
61
62 # Throws an error in first forward pass.
63 with pytest.raises(RAISED_ERROR):
64 - async for _ in client.generate(prompt="Hello my name is",
64 + async for _ in client.generate(inputs="Hello my name is",
65 sampling_params=SamplingParams(),
66 request_id=uuid.uuid4()):
67 pass
@@ -69,7 +69,7 @@ async def test_evil_forward(tmp_socket):
69
70 # Engine is errored, should get ENGINE_DEAD_ERROR.
71 with pytest.raises(MQEngineDeadError):
72 - async for _ in client.generate(prompt="Hello my name is",
72 + async for _ in client.generate(inputs="Hello my name is",
73 sampling_params=SamplingParams(),
74 request_id=uuid.uuid4()):
75 pass
@@ -118,7 +118,7 @@ async def test_failed_health_check(tmp_socket):
118
119 # Generate call should throw ENGINE_DEAD_ERROR
120 with pytest.raises(MQEngineDeadError):
121 - async for _ in client.generate(prompt="Hello my name is",
121 + async for _ in client.generate(inputs="Hello my name is",
122 sampling_params=SamplingParams(),
123 request_id=uuid.uuid4()):
124 pass
@@ -165,7 +165,7 @@ async def test_failed_abort(tmp_socket):
165 # with reference to the original KeyError("foo")
166 with pytest.raises(MQEngineDeadError) as execinfo:
167 async for _ in client.generate(
168 - prompt="Hello my name is",
168 + inputs="Hello my name is",
169 sampling_params=SamplingParams(max_tokens=2000),
170 request_id=uuid.uuid4()):
171 pass
@@ -190,7 +190,7 @@ async def test_bad_request(tmp_socket):
190
191 # Invalid request should fail, but not crash the server.
192 with pytest.raises(ValueError):
193 - async for _ in client.generate(prompt="Hello my name is",
193 + async for _ in client.generate(inputs="Hello my name is",
194 sampling_params=SamplingParams(),
195 request_id="abcd-1",
196 lora_request=LoRARequest(
@@ -199,7 +199,7 @@ async def test_bad_request(tmp_socket):
199 pass
200
201 # This request should be okay.
202 - async for _ in client.generate(prompt="Hello my name is",
202 + async for _ in client.generate(inputs="Hello my name is",
203 sampling_params=SamplingParams(),
204 request_id="abcd-2"):
205 pass
tests/mq_llm_engine/utils.py
+1 -1
@@ -20,7 +20,7 @@ async def generate(
20 count = 0
21 async for out in client.generate(
22 request_id=request_id,
23 - prompt="Hello my name is Robert and",
23 + inputs="Hello my name is Robert and",
24 sampling_params=SamplingParams(max_tokens=num_tokens,
25 temperature=0)):
26
vllm/__init__.py
+2 -2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 -from vllm.inputs import PromptType, TextPrompt, TokensPrompt
8 +from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11 EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19 "__version_tuple__",
20 "LLM",
21 "ModelRegistry",
22 - "PromptType",
22 + "PromptInputs",
23 "TextPrompt",
24 "TokensPrompt",
25 "SamplingParams",
vllm/engine/async_llm_engine.py
+13 -11
@@ -17,7 +17,7 @@ from vllm.engine.metrics_types import StatLoggerBase
17 from vllm.executor.executor_base import ExecutorAsyncBase
18 from vllm.executor.gpu_executor import GPUExecutorAsync
19 from vllm.executor.ray_utils import initialize_ray_cluster
20 -from vllm.inputs import PromptType
20 +from vllm.inputs import PromptInputs
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -405,7 +405,7 @@ class _AsyncLLMEngine(LLMEngine):
405 async def add_request_async(
406 self,
407 request_id: str,
408 - prompt: PromptType,
408 + inputs: PromptInputs,
409 params: Union[SamplingParams, PoolingParams],
410 arrival_time: Optional[float] = None,
411 lora_request: Optional[LoRARequest] = None,
@@ -420,7 +420,7 @@ class _AsyncLLMEngine(LLMEngine):
420 arrival_time = time.time()
421
422 preprocessed_inputs = await self.input_preprocessor.preprocess_async(
423 - prompt,
423 + inputs,
424 request_id=request_id,
425 lora_request=lora_request,
426 prompt_adapter_request=prompt_adapter_request,
@@ -777,7 +777,7 @@ class AsyncLLMEngine:
777 async def add_request(
778 self,
779 request_id: str,
780 - prompt: PromptType,
780 + inputs: PromptInputs,
781 params: Union[SamplingParams, PoolingParams],
782 arrival_time: Optional[float] = None,
783 lora_request: Optional[LoRARequest] = None,
@@ -797,7 +797,7 @@ class AsyncLLMEngine:
797 stream = self._request_tracker.add_request(
798 request_id,
799 verbose=self.log_requests,
800 - prompt=prompt,
800 + inputs=inputs,
801 params=params,
802 arrival_time=arrival_time or time.time(),
803 lora_request=lora_request,
@@ -808,7 +808,7 @@ class AsyncLLMEngine:
808
809 async def generate(
810 self,
811 - prompt: PromptType,
811 + inputs: PromptInputs,
812 sampling_params: SamplingParams,
813 request_id: str,
814 lora_request: Optional[LoRARequest] = None,
@@ -822,7 +822,8 @@ class AsyncLLMEngine:
822 from the LLMEngine to the caller.
823
824 Args:
825 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
825 + inputs: The inputs to the LLM. See
826 + :class:`~vllm.inputs.PromptInputs`
827 for more details about the format of each input.
828 sampling_params: The sampling parameters of the request.
829 request_id: The unique id of the request.
@@ -880,7 +881,7 @@ class AsyncLLMEngine:
881 """
882 async for output in await self.add_request(
883 request_id,
883 - prompt,
884 + inputs,
885 sampling_params,
886 lora_request=lora_request,
887 trace_headers=trace_headers,
@@ -890,7 +891,7 @@ class AsyncLLMEngine:
891
892 async def encode(
893 self,
893 - prompt: PromptType,
894 + inputs: PromptInputs,
895 pooling_params: PoolingParams,
896 request_id: str,
897 lora_request: Optional[LoRARequest] = None,
@@ -903,7 +904,8 @@ class AsyncLLMEngine:
904 from the LLMEngine to the caller.
905
906 Args:
906 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
907 + inputs: The inputs to the LLM. See
908 + :class:`~vllm.inputs.PromptInputs`
909 for more details about the format of each input.
910 pooling_params: The pooling parameters of the request.
911 request_id: The unique id of the request.
@@ -957,7 +959,7 @@ class AsyncLLMEngine:
959 """
960 async for output in await self.add_request(
961 request_id,
960 - prompt,
962 + inputs,
963 pooling_params,
964 lora_request=lora_request,
965 trace_headers=trace_headers,
vllm/engine/llm_engine.py
+5 -4
@@ -29,7 +29,7 @@ from vllm.executor.executor_base import ExecutorBase
29 from vllm.executor.gpu_executor import GPUExecutor
30 from vllm.executor.ray_utils import initialize_ray_cluster
31 from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32 - InputRegistry, LLMInputs, PromptType)
32 + InputRegistry, LLMInputs, PromptInputs)
33 from vllm.inputs.preprocess import InputPreprocessor
34 from vllm.logger import init_logger
35 from vllm.lora.request import LoRARequest
@@ -689,7 +689,7 @@ class LLMEngine:
689 def add_request(
690 self,
691 request_id: str,
692 - prompt: PromptType,
692 + inputs: PromptInputs,
693 params: Union[SamplingParams, PoolingParams],
694 arrival_time: Optional[float] = None,
695 lora_request: Optional[LoRARequest] = None,
@@ -704,7 +704,8 @@ class LLMEngine:
704
705 Args:
706 request_id: The unique ID of the request.
707 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
707 + inputs: The inputs to the LLM. See
708 + :class:`~vllm.inputs.PromptInputs`
709 for more details about the format of each input.
710 params: Parameters for sampling or pooling.
711 :class:`~vllm.SamplingParams` for text generation.
@@ -744,7 +745,7 @@ class LLMEngine:
745 arrival_time = time.time()
746
747 preprocessed_inputs = self.input_preprocessor.preprocess(
747 - prompt,
748 + inputs,
749 request_id=request_id,
750 lora_request=lora_request,
751 prompt_adapter_request=prompt_adapter_request,
vllm/engine/multiprocessing/__init__.py
+2 -2
@@ -3,7 +3,7 @@ from enum import Enum
3 from typing import List, Mapping, Optional, Union
4
5 from vllm import PoolingParams
6 -from vllm.inputs import PromptType
6 +from vllm.inputs import PromptInputs
7 from vllm.lora.request import LoRARequest
8 from vllm.outputs import RequestOutput
9 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -23,7 +23,7 @@ class MQEngineDeadError(RuntimeError):
23
24 @dataclass
25 class RPCProcessRequest:
26 - prompt: PromptType
26 + inputs: PromptInputs
27 params: Union[SamplingParams, PoolingParams]
28 request_id: str
29 lora_request: Optional[LoRARequest] = None
vllm/engine/multiprocessing/client.py
+11 -9
@@ -25,7 +25,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25 RPCStartupResponse)
26 # yapf: enable
27 from vllm.envs import VLLM_RPC_TIMEOUT
28 -from vllm.inputs import PromptType
28 +from vllm.inputs import PromptInputs
29 from vllm.logger import init_logger
30 from vllm.lora.request import LoRARequest
31 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -375,7 +375,7 @@ class MQLLMEngineClient:
375
376 def generate(
377 self,
378 - prompt: PromptType,
378 + inputs: PromptInputs,
379 sampling_params: SamplingParams,
380 request_id: str,
381 lora_request: Optional[LoRARequest] = None,
@@ -389,7 +389,8 @@ class MQLLMEngineClient:
389 from the LLMEngine to the caller.
390
391 Args:
392 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
392 + inputs: The inputs to the LLM. See
393 + :class:`~vllm.inputs.PromptInputs`
394 for more details about the format of each input.
395 sampling_params: The sampling parameters of the request.
396 request_id: The unique id of the request.
@@ -398,13 +399,13 @@ class MQLLMEngineClient:
399 prompt_adapter_request: Prompt Adapter request to use
400 for generation, if any.
401 """
401 - return self._process_request(prompt, sampling_params, request_id,
402 + return self._process_request(inputs, sampling_params, request_id,
403 lora_request, trace_headers,
404 prompt_adapter_request)
405
406 def encode(
407 self,
407 - prompt: PromptType,
408 + inputs: PromptInputs,
409 pooling_params: PoolingParams,
410 request_id: str,
411 lora_request: Optional[LoRARequest] = None,
@@ -417,7 +418,8 @@ class MQLLMEngineClient:
418 from the LLMEngine to the caller.
419
420 Args:
420 - prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
421 + inputs: The inputs to the LLM. See
422 + :class:`~vllm.inputs.PromptInputs`
423 for more details about the format of each input.
424 pooling_params: The pooling parameters of the request.
425 request_id: The unique id of the request.
@@ -428,12 +430,12 @@ class MQLLMEngineClient:
430 The output `EmbeddingRequestOutput` objects from the LLMEngine
431 for the request.
432 """
431 - return self._process_request(prompt, pooling_params, request_id,
433 + return self._process_request(inputs, pooling_params, request_id,
434 lora_request, trace_headers)
435
436 async def _process_request(
437 self,
436 - prompt: PromptType,
438 + inputs: PromptInputs,
439 params: Union[SamplingParams, PoolingParams],
440 request_id: str,
441 lora_request: Optional[LoRARequest] = None,
@@ -466,7 +468,7 @@ class MQLLMEngineClient:
468
469 request_bytes = pickle.dumps(
470 RPCProcessRequest(
469 - prompt=prompt,
471 + inputs=inputs,
472 params=params,
473 request_id=request_id,
474 lora_request=lora_request,
vllm/engine/multiprocessing/engine.py
+1 -1
@@ -252,7 +252,7 @@ class MQLLMEngine:
252 try:
253 self.engine.add_request(
254 request_id=request_id,
255 - prompt=request.prompt,
255 + inputs=request.inputs,
256 params=request.params,
257 lora_request=request.lora_request,
258 trace_headers=request.trace_headers,
vllm/engine/protocol.py
+4 -4
@@ -3,7 +3,7 @@ from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
3
4 from vllm.config import DecodingConfig, ModelConfig
5 from vllm.core.scheduler import SchedulerOutputs
6 -from vllm.inputs.data import PromptType
6 +from vllm.inputs.data import PromptInputs
7 from vllm.lora.request import LoRARequest
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -35,19 +35,19 @@ class EngineClient(Protocol):
35
36 def generate(
37 self,
38 - prompt: PromptType,
38 + inputs: PromptInputs,
39 sampling_params: SamplingParams,
40 request_id: str,
41 lora_request: Optional[LoRARequest] = None,
42 trace_headers: Optional[Mapping[str, str]] = None,
43 prompt_adapter_request: Optional[PromptAdapterRequest] = None
44 ) -> AsyncGenerator[RequestOutput, None]:
45 - """Generate outputs for a request."""
45 + """Generates outputs for a request"""
46 ...
47
48 def encode(
49 self,
50 - prompt: PromptType,
50 + inputs: PromptInputs,
51 pooling_params: PoolingParams,
52 request_id: str,
53 lora_request: Optional[LoRARequest] = None,
vllm/entrypoints/llm.py
+38 -42
@@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
12 apply_hf_chat_template,
13 apply_mistral_chat_template,
14 parse_chat_messages)
15 -from vllm.inputs import PromptType, TextPrompt, TokensPrompt
15 +from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
16 from vllm.inputs.parse import parse_and_batch_prompt
17 from vllm.logger import init_logger
18 from vllm.lora.request import LoRARequest
@@ -293,8 +293,8 @@ class LLM:
293 @overload
294 def generate(
295 self,
296 - prompts: Union[PromptType, Sequence[PromptType]],
297 - /,
296 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
297 + /, # We may enable `inputs` keyword after removing the old API
298 *,
299 sampling_params: Optional[Union[SamplingParams,
300 Sequence[SamplingParams]]] = None,
@@ -311,7 +311,7 @@ class LLM:
311 )
312 def generate(
313 self,
314 - prompts: Union[Union[PromptType, Sequence[PromptType]],
314 + prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
315 Optional[Union[str, List[str]]]] = None,
316 sampling_params: Optional[Union[SamplingParams,
317 Sequence[SamplingParams]]] = None,
@@ -329,9 +329,7 @@ class LLM:
329 into a single list and pass it to this method.
330
331 Args:
332 - prompts: The prompts to the LLM. You may pass a sequence of prompts
333 - for batch inference. See :class:`~vllm.inputs.PromptType`
334 - for more details about the format of each prompts.
332 + inputs: A list of inputs to generate completions for.
333 sampling_params: The sampling parameters for text generation. If
334 None, we use the default sampling parameters.
335 When it is a single value, it is applied to every prompt.
@@ -357,13 +355,12 @@ class LLM:
355 "models (XForCausalLM, XForConditionalGeneration).")
356
357 if prompt_token_ids is not None:
360 - parsed_prompts = self._convert_v1_inputs(
358 + inputs = self._convert_v1_inputs(
359 prompts=cast(Optional[Union[str, List[str]]], prompts),
360 prompt_token_ids=prompt_token_ids,
361 )
362 else:
365 - parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
366 - prompts)
363 + inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
364
365 if isinstance(guided_options_request, dict):
366 if len(guided_options_request) > 1:
@@ -378,7 +375,7 @@ class LLM:
375 sampling_params = SamplingParams()
376
377 self._validate_and_add_requests(
381 - prompts=parsed_prompts,
378 + inputs=inputs,
379 params=sampling_params,
380 lora_request=lora_request,
381 prompt_adapter_request=prompt_adapter_request,
@@ -533,9 +530,9 @@ class LLM:
530 conversation, mm_data = parse_chat_messages(messages, model_config,
531 tokenizer)
532
536 - prompt_data: Union[str, List[int]]
533 + prompt: Union[str, List[int]]
534 if isinstance(tokenizer, MistralTokenizer):
538 - prompt_data = apply_mistral_chat_template(
535 + prompt = apply_mistral_chat_template(
536 tokenizer,
537 messages=messages,
538 chat_template=chat_template,
@@ -543,7 +540,7 @@ class LLM:
540 tools=tools,
541 )
542 else:
546 - prompt_data = apply_hf_chat_template(
543 + prompt = apply_hf_chat_template(
544 tokenizer,
545 conversation=conversation,
546 chat_template=chat_template,
@@ -551,17 +548,17 @@ class LLM:
548 tools=tools,
549 )
550
554 - prompt: PromptType
555 - if is_list_of(prompt_data, int):
556 - prompt = TokensPrompt(prompt_token_ids=prompt_data)
551 + inputs: PromptInputs
552 + if is_list_of(prompt, int):
553 + inputs = TokensPrompt(prompt_token_ids=prompt)
554 else:
558 - prompt = TextPrompt(prompt=prompt_data)
555 + inputs = TextPrompt(prompt=prompt)
556
557 if mm_data is not None:
561 - prompt["multi_modal_data"] = mm_data
558 + inputs["multi_modal_data"] = mm_data
559
560 return self.generate(
564 - prompt,
561 + inputs,
562 sampling_params=sampling_params,
563 use_tqdm=use_tqdm,
564 lora_request=lora_request,
@@ -631,8 +628,8 @@ class LLM:
628 @overload
629 def encode(
630 self,
634 - prompts: Union[PromptType, Sequence[PromptType]],
635 - /,
631 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
632 + /, # We may enable `inputs` keyword after removing the old API
633 *,
634 pooling_params: Optional[Union[PoolingParams,
635 Sequence[PoolingParams]]] = None,
@@ -649,7 +646,7 @@ class LLM:
646 )
647 def encode(
648 self,
652 - prompts: Union[Union[PromptType, Sequence[PromptType]],
649 + prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
650 Optional[Union[str, List[str]]]] = None,
651 pooling_params: Optional[Union[PoolingParams,
652 Sequence[PoolingParams]]] = None,
@@ -665,9 +662,9 @@ class LLM:
662 into a single list and pass it to this method.
663
664 Args:
668 - prompts: The prompts to the LLM. You may pass a sequence of prompts
669 - for batch inference. See :class:`~vllm.inputs.PromptType`
670 - for more details about the format of each prompts.
665 + inputs: The inputs to the LLM. You may pass a sequence of inputs for
666 + batch inference. See :class:`~vllm.inputs.PromptInputs`
667 + for more details about the format of each input.
668 pooling_params: The pooling parameters for pooling. If None, we
669 use the default pooling parameters.
670 use_tqdm: Whether to use tqdm to display the progress bar.
@@ -690,20 +687,19 @@ class LLM:
687 )
688
689 if prompt_token_ids is not None:
693 - parsed_prompts = self._convert_v1_inputs(
690 + inputs = self._convert_v1_inputs(
691 prompts=cast(Optional[Union[str, List[str]]], prompts),
692 prompt_token_ids=prompt_token_ids,
693 )
694 else:
698 - parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
699 - prompts)
695 + inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
696
697 if pooling_params is None:
698 # Use default pooling params.
699 pooling_params = PoolingParams()
700
701 self._validate_and_add_requests(
706 - prompts=parsed_prompts,
702 + inputs=inputs,
703 params=pooling_params,
704 lora_request=lora_request,
705 prompt_adapter_request=prompt_adapter_request,
@@ -747,9 +743,9 @@ class LLM:
743 raise ValueError("Either prompts or prompt_token_ids must be "
744 "provided.")
745
750 - parsed_prompts: List[PromptType] = []
746 + inputs: List[PromptInputs] = []
747 for i in range(num_requests):
752 - item: PromptType
748 + item: PromptInputs
749
750 if prompts is not None:
751 item = TextPrompt(prompt=prompts[i])
@@ -758,24 +754,24 @@ class LLM:
754 else:
755 raise AssertionError
756
761 - parsed_prompts.append(item)
757 + inputs.append(item)
758
763 - return parsed_prompts
759 + return inputs
760
761 def _validate_and_add_requests(
762 self,
767 - prompts: Union[PromptType, Sequence[PromptType]],
763 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
764 params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
765 Sequence[PoolingParams]],
766 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
767 prompt_adapter_request: Optional[PromptAdapterRequest],
768 guided_options: Optional[GuidedDecodingRequest] = None,
769 ) -> None:
774 - if isinstance(prompts, (str, dict)):
770 + if isinstance(inputs, (str, dict)):
771 # Convert a single prompt to a list.
776 - prompts = [prompts]
772 + inputs = [inputs]
773
778 - num_requests = len(prompts)
774 + num_requests = len(inputs)
775 if isinstance(params, list) and len(params) != num_requests:
776 raise ValueError("The lengths of prompts and params "
777 "must be the same.")
@@ -792,9 +788,9 @@ class LLM:
788 sp.output_kind = RequestOutputKind.FINAL_ONLY
789
790 # Add requests to the engine.
795 - for i, prompt in enumerate(prompts):
791 + for i, request_inputs in enumerate(inputs):
792 self._add_request(
797 - prompt,
793 + request_inputs,
794 params[i] if isinstance(params, Sequence) else params,
795 lora_request=lora_request[i] if isinstance(
796 lora_request, Sequence) else lora_request,
@@ -803,7 +799,7 @@ class LLM:
799
800 def _add_request(
801 self,
806 - prompt: PromptType,
802 + inputs: PromptInputs,
803 params: Union[SamplingParams, PoolingParams],
804 lora_request: Optional[LoRARequest] = None,
805 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -811,7 +807,7 @@ class LLM:
807 request_id = str(next(self.request_counter))
808 self.llm_engine.add_request(
809 request_id,
814 - prompt,
810 + inputs,
811 params,
812 lora_request=lora_request,
813 prompt_adapter_request=prompt_adapter_request,
vllm/inputs/__init__.py
+3 -3
@@ -1,5 +1,5 @@
1 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2 - LLMInputs, PromptType, SingletonPrompt, TextPrompt,
2 + LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
3 TokensPrompt, build_explicit_enc_dec_prompt,
4 to_enc_dec_tuple_list, zip_enc_dec_prompts)
5 from .registry import InputContext, InputRegistry
@@ -16,8 +16,8 @@ See also:
16 __all__ = [
17 "TextPrompt",
18 "TokensPrompt",
19 - "PromptType",
20 - "SingletonPrompt",
19 + "PromptInputs",
20 + "SingletonPromptInputs",
21 "ExplicitEncoderDecoderPrompt",
22 "LLMInputs",
23 "EncoderDecoderLLMInputs",
vllm/inputs/data.py
+15 -11
@@ -33,7 +33,7 @@ class TokensPrompt(TypedDict):
33 """
34
35
36 -SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
36 +SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
37 """
38 Set of possible schemas for a single LLM input:
39
@@ -46,7 +46,7 @@ which may be utilized for encoder/decoder models when
46 the user desires to express both the encoder & decoder
47 prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
49 -A prompt of type :class:`SingletonPromptType` may be employed
49 +A prompt of type :class:`SingletonPromptInputs` may be employed
50 as (1) input to a decoder-only model, (2) input to
51 the encoder of an encoder/decoder model, in the scenario
52 where the decoder-prompt is not specified explicitly, or
@@ -55,12 +55,12 @@ more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55 """
56
57 _T1_co = TypeVar("_T1_co",
58 - bound=SingletonPrompt,
59 - default=SingletonPrompt,
58 + bound=SingletonPromptInputs,
59 + default=SingletonPromptInputs,
60 covariant=True)
61 _T2_co = TypeVar("_T2_co",
62 - bound=SingletonPrompt,
63 - default=SingletonPrompt,
62 + bound=SingletonPromptInputs,
63 + default=SingletonPromptInputs,
64 covariant=True)
65
66
@@ -72,7 +72,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
72
73 The encoder and decoder prompts, respectively,
74 may formatted according to any of the
75 - :class:`SingletonPromptType` schemas, and are not
75 + :class:`SingletonPromptInputs` schemas, and are not
76 required to have the same schema.
77
78 Only the encoder prompt may have multi-modal data.
@@ -81,7 +81,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
81 be used as an input to a decoder-only model,
82 and that the `encoder_prompt` and `decoder_prompt`
83 fields of this data structure themselves must be
84 - :class:`SingletonPromptType` instances.
84 + :class:`SingletonPromptInputs` instances.
85 """
86
87 encoder_prompt: _T1_co
@@ -89,7 +89,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
89 decoder_prompt: Optional[_T2_co]
90
91
92 -PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
92 +PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
93 """
94 Set of possible schemas for an LLM input, including
95 both decoder-only and encoder/decoder input types:
@@ -140,8 +140,12 @@ class EncoderDecoderLLMInputs(LLMInputs):
140 """
141
142
143 -_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
144 -_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
143 +_T1 = TypeVar("_T1",
144 + bound=SingletonPromptInputs,
145 + default=SingletonPromptInputs)
146 +_T2 = TypeVar("_T2",
147 + bound=SingletonPromptInputs,
148 + default=SingletonPromptInputs)
149
150
151 def build_explicit_enc_dec_prompt(
vllm/inputs/parse.py
+11 -11
@@ -5,7 +5,7 @@ from typing_extensions import TypeIs
5 from vllm.utils import is_list_of
6
7 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8 - LLMInputs, PromptType, SingletonPrompt, TextPrompt,
8 + LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
9 TokensPrompt)
10
11
@@ -81,23 +81,23 @@ class ParsedTokensPrompt(TypedDict):
81
82
83 def parse_singleton_prompt(
84 - prompt: SingletonPrompt,
84 + inputs: SingletonPromptInputs,
85 ) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86 - if isinstance(prompt, str):
87 - return ParsedStrPrompt(type="str", content=prompt)
88 - elif isinstance(prompt, dict):
89 - if "prompt_token_ids" in prompt:
86 + if isinstance(inputs, str):
87 + return ParsedStrPrompt(type="str", content=inputs)
88 + elif isinstance(inputs, dict):
89 + if "prompt_token_ids" in inputs:
90 return ParsedTokensPrompt(type="tokens",
91 - content=prompt) # type: ignore
92 - elif "prompt" in prompt:
93 - return ParsedTextPrompt(type="text", content=prompt)
91 + content=inputs) # type: ignore
92 + elif "prompt" in inputs:
93 + return ParsedTextPrompt(type="text", content=inputs)
94
95 raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98 def is_explicit_encoder_decoder_prompt(
99 - prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 - return isinstance(prompt, dict) and "encoder_prompt" in prompt
99 + inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 + return isinstance(inputs, dict) and "encoder_prompt" in inputs
101
102
103 def is_valid_encoder_decoder_llm_inputs(
vllm/inputs/preprocess.py
+43 -43
@@ -9,8 +9,8 @@ from vllm.lora.request import LoRARequest
9 from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
12 -from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
13 - SingletonPrompt)
12 +from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
13 + SingletonPromptInputs)
14 from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
15
16 if TYPE_CHECKING:
@@ -206,7 +206,7 @@ class InputPreprocessor:
206
207 def _extract_prompt_components(
208 self,
209 - prompt: SingletonPrompt,
209 + inputs: SingletonPromptInputs,
210 request_id: str,
211 lora_request: Optional[LoRARequest] = None,
212 ) -> PromptComponents:
@@ -216,7 +216,7 @@ class InputPreprocessor:
216 Arguments:
217
218 * request_id
219 - * prompt: single encoder or decoder input prompt
219 + * inputs: single encoder or decoder input prompt
220 * lora_request: this is only valid for decoder prompts
221
222 Returns:
@@ -226,24 +226,24 @@ class InputPreprocessor:
226 * multi_modal_data
227 '''
228
229 - parsed = parse_singleton_prompt(prompt)
229 + parsed = parse_singleton_prompt(inputs)
230
231 if parsed["type"] == "str":
232 - prompt_text = parsed["content"]
232 + prompt = parsed["content"]
233 prompt_token_ids = self._tokenize_prompt(
234 - prompt_text,
234 + prompt,
235 request_id=request_id,
236 lora_request=lora_request,
237 )
238 multi_modal_data = None
239 elif parsed["type"] == "tokens":
240 - prompt_text = None
240 + prompt = None
241 prompt_token_ids = parsed["content"]["prompt_token_ids"]
242 multi_modal_data = parsed["content"].get("multi_modal_data")
243 elif parsed["type"] == "text":
244 - prompt_text = parsed["content"]["prompt"]
244 + prompt = parsed["content"]["prompt"]
245 prompt_token_ids = self._tokenize_prompt(
246 - prompt_text,
246 + prompt,
247 request_id=request_id,
248 lora_request=lora_request,
249 )
@@ -251,33 +251,33 @@ class InputPreprocessor:
251 else:
252 assert_never(parsed)
253
254 - return prompt_text, prompt_token_ids, multi_modal_data
254 + return prompt, prompt_token_ids, multi_modal_data
255
256 async def _extract_prompt_components_async(
257 self,
258 - prompt: SingletonPrompt,
258 + inputs: SingletonPromptInputs,
259 request_id: str,
260 lora_request: Optional[LoRARequest] = None,
261 ) -> PromptComponents:
262 """Async version of :meth:`_extract_prompt_components`."""
263 - parsed = parse_singleton_prompt(prompt)
263 + parsed = parse_singleton_prompt(inputs)
264
265 if parsed["type"] == "str":
266 - prompt_text = parsed["content"]
266 + prompt = parsed["content"]
267 prompt_token_ids = await self._tokenize_prompt_async(
268 - prompt_text,
268 + prompt,
269 request_id=request_id,
270 lora_request=lora_request,
271 )
272 multi_modal_data = None
273 elif parsed["type"] == "tokens":
274 - prompt_text = None
274 + prompt = None
275 prompt_token_ids = parsed["content"]["prompt_token_ids"]
276 multi_modal_data = parsed["content"].get("multi_modal_data")
277 elif parsed["type"] == "text":
278 - prompt_text = parsed["content"]["prompt"]
278 + prompt = parsed["content"]["prompt"]
279 prompt_token_ids = await self._tokenize_prompt_async(
280 - prompt_text,
280 + prompt,
281 request_id=request_id,
282 lora_request=lora_request,
283 )
@@ -285,7 +285,7 @@ class InputPreprocessor:
285 else:
286 assert_never(parsed)
287
288 - return prompt_text, prompt_token_ids, multi_modal_data
288 + return prompt, prompt_token_ids, multi_modal_data
289
290 def _build_enc_dec_llm_inputs(
291 self,
@@ -311,7 +311,7 @@ class InputPreprocessor:
311
312 def _process_encoder_decoder_prompt(
313 self,
314 - prompt: PromptType,
314 + inputs: PromptInputs,
315 request_id: str,
316 ) -> EncoderDecoderLLMInputs:
317 '''
@@ -339,7 +339,7 @@ class InputPreprocessor:
339
340 Arguments:
341
342 - * prompt: an input prompt
342 + * inputs: an input prompt
343 * request_id
344
345 Returns:
@@ -350,13 +350,13 @@ class InputPreprocessor:
350 encoder_comps: PromptComponents
351 decoder_comps: DecoderPromptComponents
352
353 - if is_explicit_encoder_decoder_prompt(prompt):
353 + if is_explicit_encoder_decoder_prompt(inputs):
354 encoder_comps = self._extract_prompt_components(
355 - prompt["encoder_prompt"],
355 + inputs["encoder_prompt"],
356 request_id=request_id,
357 )
358
359 - if (decoder_input := prompt["decoder_prompt"]) is None:
359 + if (decoder_input := inputs["decoder_prompt"]) is None:
360 decoder_comps = None, None, None
361 else:
362 decoder_comps = self._extract_prompt_components(
@@ -365,7 +365,7 @@ class InputPreprocessor:
365 )
366 else:
367 encoder_comps = self._extract_prompt_components(
368 - prompt,
368 + inputs,
369 request_id=request_id,
370 )
371
@@ -375,20 +375,20 @@ class InputPreprocessor:
375
376 async def _process_encoder_decoder_prompt_async(
377 self,
378 - prompt: PromptType,
378 + inputs: PromptInputs,
379 request_id: str,
380 ) -> EncoderDecoderLLMInputs:
381 """Async version of :meth:`_process_encoder_decoder_prompt`."""
382 encoder_comps: PromptComponents
383 decoder_comps: DecoderPromptComponents
384
385 - if is_explicit_encoder_decoder_prompt(prompt):
385 + if is_explicit_encoder_decoder_prompt(inputs):
386 encoder_task = self._extract_prompt_components_async(
387 - prompt["encoder_prompt"],
387 + inputs["encoder_prompt"],
388 request_id=request_id,
389 )
390
391 - if (decoder_input := prompt["decoder_prompt"]) is None:
391 + if (decoder_input := inputs["decoder_prompt"]) is None:
392 encoder_comps = await encoder_task
393 decoder_comps = None, None, None
394 else:
@@ -401,7 +401,7 @@ class InputPreprocessor:
401 encoder_task, decoder_task)
402 else:
403 encoder_comps = await self._extract_prompt_components_async(
404 - prompt,
404 + inputs,
405 request_id=request_id,
406 )
407
@@ -425,7 +425,7 @@ class InputPreprocessor:
425
426 def _process_decoder_only_prompt(
427 self,
428 - prompt: SingletonPrompt,
428 + inputs: SingletonPromptInputs,
429 request_id: str,
430 lora_request: Optional[LoRARequest] = None,
431 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -436,7 +436,7 @@ class InputPreprocessor:
436
437 Arguments:
438
439 - * prompt: input prompt
439 + * inputs: input prompt
440 * request_id
441 * lora_request
442 * prompt_adapter_request
@@ -447,7 +447,7 @@ class InputPreprocessor:
447 '''
448
449 prompt_comps = self._extract_prompt_components(
450 - prompt,
450 + inputs,
451 request_id=request_id,
452 lora_request=lora_request,
453 )
@@ -459,14 +459,14 @@ class InputPreprocessor:
459
460 async def _process_decoder_only_prompt_async(
461 self,
462 - prompt: SingletonPrompt,
462 + inputs: SingletonPromptInputs,
463 request_id: str,
464 lora_request: Optional[LoRARequest] = None,
465 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
466 ) -> LLMInputs:
467 """Async version of :meth:`_process_decoder_only_prompt`."""
468 prompt_comps = await self._extract_prompt_components_async(
469 - prompt,
469 + inputs,
470 request_id=request_id,
471 lora_request=lora_request,
472 )
@@ -478,7 +478,7 @@ class InputPreprocessor:
478
479 def preprocess(
480 self,
481 - prompt: PromptType,
481 + inputs: PromptInputs,
482 request_id: str,
483 lora_request: Optional[LoRARequest] = None,
484 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -488,17 +488,17 @@ class InputPreprocessor:
488 # Encoder-decoder model requires special mapping of
489 # input prompts to encoder & decoder
490 return self._process_encoder_decoder_prompt(
491 - prompt,
491 + inputs,
492 request_id=request_id,
493 )
494
495 - if is_explicit_encoder_decoder_prompt(prompt):
495 + if is_explicit_encoder_decoder_prompt(inputs):
496 raise ValueError("Cannot pass encoder-decoder prompt "
497 "to decoder-only models")
498
499 # Decoder-only operation
500 return self._process_decoder_only_prompt(
501 - prompt,
501 + inputs,
502 request_id=request_id,
503 lora_request=lora_request,
504 prompt_adapter_request=prompt_adapter_request,
@@ -506,7 +506,7 @@ class InputPreprocessor:
506
507 async def preprocess_async(
508 self,
509 - prompt: PromptType,
509 + inputs: PromptInputs,
510 request_id: str,
511 lora_request: Optional[LoRARequest] = None,
512 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -516,17 +516,17 @@ class InputPreprocessor:
516 # Encoder-decoder model requires special mapping of
517 # input prompts to encoder & decoder
518 return await self._process_encoder_decoder_prompt_async(
519 - prompt,
519 + inputs,
520 request_id=request_id,
521 )
522
523 - if is_explicit_encoder_decoder_prompt(prompt):
523 + if is_explicit_encoder_decoder_prompt(inputs):
524 raise ValueError("Cannot pass encoder-decoder prompt "
525 "to decoder-only models")
526
527 # Decoder-only operation
528 return await self._process_decoder_only_prompt_async(
529 - prompt,
529 + inputs,
530 request_id=request_id,
531 lora_request=lora_request,
532 prompt_adapter_request=prompt_adapter_request,